From 05bd614ae7e43dc1bd2c62ca4ee5ed9e4a7278af Mon Sep 17 00:00:00 2001 From: ModelHub XC Date: Tue, 16 Jun 2026 05:55:17 +0800 Subject: [PATCH] =?UTF-8?q?=E5=88=9D=E5=A7=8B=E5=8C=96=E9=A1=B9=E7=9B=AE?= =?UTF-8?q?=EF=BC=8C=E7=94=B1ModelHub=20XC=E7=A4=BE=E5=8C=BA=E6=8F=90?= =?UTF-8?q?=E4=BE=9B=E6=A8=A1=E5=9E=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Model: oxdev/security-auditor-grpo Source: Original Platform --- .gitattributes | 36 + README.md | 165 +++ chat_template.jinja | 54 + checkpoint-300/chat_template.jinja | 54 + checkpoint-300/config.json | 57 + checkpoint-300/generation_config.json | 13 + checkpoint-300/model.safetensors | 3 + checkpoint-300/optimizer.pt | 3 + checkpoint-300/rng_state.pth | 3 + checkpoint-300/scheduler.pt | 3 + checkpoint-300/tokenizer.json | 3 + checkpoint-300/tokenizer_config.json | 32 + checkpoint-300/trainer_state.json | 1803 +++++++++++++++++++++++ checkpoint-300/training_args.bin | 3 + checkpoint-326/chat_template.jinja | 54 + checkpoint-326/config.json | 57 + checkpoint-326/generation_config.json | 13 + checkpoint-326/model.safetensors | 3 + checkpoint-326/optimizer.pt | 3 + checkpoint-326/rng_state.pth | 3 + checkpoint-326/scheduler.pt | 3 + checkpoint-326/tokenizer.json | 3 + checkpoint-326/tokenizer_config.json | 32 + checkpoint-326/trainer_state.json | 1948 +++++++++++++++++++++++++ checkpoint-326/training_args.bin | 3 + config.json | 57 + generation_config.json | 13 + model.safetensors | 3 + tokenizer.json | 3 + tokenizer_config.json | 32 + train_grpo_job.py | 252 ++++ train_grpo_v2.py | 356 +++++ train_grpo_v2_colab.ipynb | 482 ++++++ training_args.bin | 3 + 34 files changed, 5555 insertions(+) create mode 100644 .gitattributes create mode 100644 README.md create mode 100644 chat_template.jinja create mode 100644 checkpoint-300/chat_template.jinja create mode 100644 checkpoint-300/config.json create mode 100644 checkpoint-300/generation_config.json create mode 100644 checkpoint-300/model.safetensors create mode 100644 checkpoint-300/optimizer.pt create mode 100644 checkpoint-300/rng_state.pth create mode 100644 checkpoint-300/scheduler.pt create mode 100644 checkpoint-300/tokenizer.json create mode 100644 checkpoint-300/tokenizer_config.json create mode 100644 checkpoint-300/trainer_state.json create mode 100644 checkpoint-300/training_args.bin create mode 100644 checkpoint-326/chat_template.jinja create mode 100644 checkpoint-326/config.json create mode 100644 checkpoint-326/generation_config.json create mode 100644 checkpoint-326/model.safetensors create mode 100644 checkpoint-326/optimizer.pt create mode 100644 checkpoint-326/rng_state.pth create mode 100644 checkpoint-326/scheduler.pt create mode 100644 checkpoint-326/tokenizer.json create mode 100644 checkpoint-326/tokenizer_config.json create mode 100644 checkpoint-326/trainer_state.json create mode 100644 checkpoint-326/training_args.bin create mode 100644 config.json create mode 100644 generation_config.json create mode 100644 model.safetensors create mode 100644 tokenizer.json create mode 100644 tokenizer_config.json create mode 100644 train_grpo_job.py create mode 100644 train_grpo_v2.py create mode 100644 train_grpo_v2_colab.ipynb create mode 100644 training_args.bin diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..52373fe --- /dev/null +++ b/.gitattributes @@ -0,0 +1,36 @@ +*.7z filter=lfs diff=lfs merge=lfs -text +*.arrow filter=lfs diff=lfs merge=lfs -text +*.bin filter=lfs diff=lfs merge=lfs -text +*.bz2 filter=lfs diff=lfs merge=lfs -text +*.ckpt filter=lfs diff=lfs merge=lfs -text +*.ftz filter=lfs diff=lfs merge=lfs -text +*.gz filter=lfs diff=lfs merge=lfs -text +*.h5 filter=lfs diff=lfs merge=lfs -text +*.joblib filter=lfs diff=lfs merge=lfs -text +*.lfs.* filter=lfs diff=lfs merge=lfs -text +*.mlmodel filter=lfs diff=lfs merge=lfs -text +*.model filter=lfs diff=lfs merge=lfs -text +*.msgpack filter=lfs diff=lfs merge=lfs -text +*.npy filter=lfs diff=lfs merge=lfs -text +*.npz filter=lfs diff=lfs merge=lfs -text +*.onnx filter=lfs diff=lfs merge=lfs -text +*.ot filter=lfs diff=lfs merge=lfs -text +*.parquet filter=lfs diff=lfs merge=lfs -text +*.pb filter=lfs diff=lfs merge=lfs -text +*.pickle filter=lfs diff=lfs merge=lfs -text +*.pkl filter=lfs diff=lfs merge=lfs -text +*.pt filter=lfs diff=lfs merge=lfs -text +*.pth filter=lfs diff=lfs merge=lfs -text +*.rar filter=lfs diff=lfs merge=lfs -text +*.safetensors filter=lfs diff=lfs merge=lfs -text +saved_model/**/* filter=lfs diff=lfs merge=lfs -text +*.tar.* filter=lfs diff=lfs merge=lfs -text +*.tar filter=lfs diff=lfs merge=lfs -text +*.tflite filter=lfs diff=lfs merge=lfs -text +*.tgz filter=lfs diff=lfs merge=lfs -text +*.wasm filter=lfs diff=lfs merge=lfs -text +*.xz filter=lfs diff=lfs merge=lfs -text +*.zip filter=lfs diff=lfs merge=lfs -text +*.zst filter=lfs diff=lfs merge=lfs -text +*tfevents* filter=lfs diff=lfs merge=lfs -text +tokenizer.json filter=lfs diff=lfs merge=lfs -text diff --git a/README.md b/README.md new file mode 100644 index 0000000..5ed9f60 --- /dev/null +++ b/README.md @@ -0,0 +1,165 @@ +--- +base_model: Qwen/Qwen2.5-Coder-0.5B-Instruct +library_name: transformers +model_name: security-auditor-grpo +tags: +- generated_from_trainer +- grpo +- trl +- security +- smart-contracts +- solidity +- audit +- web3 +license: apache-2.0 +datasets: +- oxdev/smart-contract-security-sft +- oxdev/smart-contract-security-audit-v2 +pipeline_tag: text-generation +language: +- en +--- + +# šŸ” Smart Contract Security Auditor (GRPO) + +A specialized **smart contract security auditor** built on [Qwen2.5-Coder-0.5B-Instruct](https://huggingface.co/Qwen/Qwen2.5-Coder-0.5B-Instruct), fine-tuned using **Group Relative Policy Optimization (GRPO)** on real-world audit findings from top security firms. + +## šŸŽÆ What It Does + +Given a Solidity smart contract, this model identifies security vulnerabilities and produces structured audit findings with: +- Vulnerability classification (reentrancy, access control, oracle manipulation, etc.) +- Severity assessment (Critical/High/Medium/Low) +- Detailed description of the vulnerability +- Impact analysis +- Proof of concept exploit code +- Recommended fixes + +## Quick Start + +```python +from transformers import AutoModelForCausalLM, AutoTokenizer, pipeline + +model = AutoModelForCausalLM.from_pretrained( + "oxdev/security-auditor-grpo", + use_cache=True, # Important: config has use_cache=False from training +) +tokenizer = AutoTokenizer.from_pretrained("oxdev/security-auditor-grpo") +pipe = pipeline("text-generation", model=model, tokenizer=tokenizer, device="cuda") + +messages = [ + {"role": "system", "content": "You are an expert smart contract security auditor. Analyze the provided Solidity code for vulnerabilities."}, + {"role": "user", "content": """Audit this contract: +```solidity +contract SimpleBank { + mapping(address => uint256) public balances; + function deposit() public payable { balances[msg.sender] += msg.value; } + function withdraw(uint256 amount) public { + require(balances[msg.sender] >= amount); + (bool success, ) = msg.sender.call{value: amount}(""); + require(success); + balances[msg.sender] -= amount; + } +} +```"""}, +] + +result = pipe(messages, max_new_tokens=512, do_sample=False, return_full_text=False) +output = result[0]["generated_text"] +if isinstance(output, list): + output = output[-1]["content"] +print(output) +``` + +## šŸ”— Try It Live + +**Interactive Demo:** [oxdev/security-auditor-demo](https://huggingface.co/spaces/oxdev/security-auditor-demo) ā€” Side-by-side comparison with base model, 7 test cases with known vulnerabilities, automated scoring. + +## Training Details + +### V1 (Current Model) +- **Method:** GRPO (Group Relative Policy Optimization) +- **Base Model:** Qwen2.5-Coder-0.5B-Instruct +- **Dataset:** [oxdev/smart-contract-security-sft](https://huggingface.co/datasets/oxdev/smart-contract-security-sft) (327 synthetic samples) +- **Hardware:** NVIDIA T4 (16GB) +- **Epochs:** 2 +- **Reward Functions:** Format compliance, finding rate +- **Results:** + - Format reward: 0.025 ā†’ 0.40 (**16Ɨ improvement**) + - Finding rate: 0% ā†’ 50-75% + - Mean reward: -0.34 ā†’ -0.006 + +### V2 (Pending ā€” Colab Notebook Ready) +- **Dataset:** [oxdev/smart-contract-security-audit-v2](https://huggingface.co/datasets/oxdev/smart-contract-security-audit-v2) (50,902 real audit findings) +- **Sources:** SkywardNomad92/smart-contract-audit-findings, samscrack/cyfrin-audit-findings, Solodit API +- **4 Reward Functions:** Format (0.25), Severity matching (0.25), Category matching (0.25), Quality (0.25) +- **Train on Colab:** Open [`train_grpo_v2_colab.ipynb`](https://huggingface.co/oxdev/security-auditor-grpo/blob/main/train_grpo_v2_colab.ipynb) in Google Colab with a free T4 GPU + +## Vulnerability Categories Covered + +| Category | Keywords | +|----------|----------| +| Reentrancy | reentrancy, reentrant, callback | +| Access Control | unauthorized, permission, onlyowner | +| Oracle Manipulation | price feed, chainlink, twap | +| Flash Loan | flash loan, flashloan | +| Overflow/Underflow | overflow, underflow, arithmetic | +| Front-running | front-run, sandwich, MEV | +| DoS | denial of service, gas limit, unbounded | +| Token Issues | fee-on-transfer, rebasing, ERC20 | +| Storage | storage collision, delegatecall, proxy | +| Cross-chain | bridge, relay, message passing | +| Liquidation | liquidation, collateral, health factor | +| Signature | ecrecover, replay, nonce, EIP712 | +| Initialization | uninitialized, constructor | +| Rounding | precision, truncation, decimal | + +## Architecture + +- **Model:** Qwen2ForCausalLM +- **Parameters:** 0.5B +- **Hidden Size:** 896 +- **Layers:** 24 +- **Attention Heads:** 14 (2 KV heads) +- **Context Length:** 32,768 tokens +- **Chat Template:** ChatML (`<|im_start|>` / `<|im_end|>`) + +## āš ļø Important Notes + +1. **Set `use_cache=True`** when loading for inference ā€” the saved config has `use_cache=False` from training, which makes generation 10-20Ɨ slower +2. **This is a 0.5B model** ā€” it's fast but not as capable as larger models. Use it for quick triage, not as a replacement for professional audits +3. **V1 was trained on 327 samples** ā€” V2 training on 50K real findings will significantly improve quality + +## Files + +| File | Description | +|------|-------------| +| `model.safetensors` | V1 trained model weights (1.8GB) | +| `train_grpo_job.py` | V1 training script | +| `train_grpo_v2.py` | V2 training script (4 reward functions) | +| `train_grpo_v2_colab.ipynb` | V2 Colab notebook (free T4 GPU) | +| `checkpoint-300/` | V1 training checkpoint | +| `checkpoint-326/` | V1 final checkpoint | + +## Related Resources + +- **GitHub:** [0xedev/skills](https://github.com/0xedev/skills) ā€” Pashov Audit Group AI-powered security skills +- **V2 Dataset:** [oxdev/smart-contract-security-audit-v2](https://huggingface.co/datasets/oxdev/smart-contract-security-audit-v2) +- **Demo Space:** [oxdev/security-auditor-demo](https://huggingface.co/spaces/oxdev/security-auditor-demo) + +## Framework Versions + +- TRL: 1.2.0 +- Transformers: 5.6.2 +- PyTorch: 2.6.0+cu126 +- Datasets: 4.8.4 + +## Citations + +```bibtex +@article{shao2024deepseekmath, + title = {{DeepSeekMath: Pushing the Limits of Mathematical Reasoning in Open Language Models}}, + author = {Zhihong Shao and Peiyi Wang and Qihao Zhu and Runxin Xu and Junxiao Song and others}, + year = 2024, + eprint = {arXiv:2402.03300}, +} +``` diff --git a/chat_template.jinja b/chat_template.jinja new file mode 100644 index 0000000..bdf7919 --- /dev/null +++ b/chat_template.jinja @@ -0,0 +1,54 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0]['role'] == 'system' %} + {{- messages[0]['content'] }} + {%- else %} + {{- 'You are Qwen, created by Alibaba Cloud. You are a helpful assistant.' }} + {%- endif %} + {{- "\n\n# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0]['role'] == 'system' %} + {{- '<|im_start|>system\n' + messages[0]['content'] + '<|im_end|>\n' }} + {%- else %} + {{- '<|im_start|>system\nYou are Qwen, created by Alibaba Cloud. You are a helpful assistant.<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- for message in messages %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) or (message.role == "assistant" and not message.tool_calls) %} + {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {{- '<|im_start|>' + message.role }} + {%- if message.content %} + {{- '\n' + message.content }} + {%- endif %} + {%- for tool_call in message.tool_calls %} + {%- if tool_call.function is defined %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {{- tool_call.arguments | tojson }} + {{- '}\n' }} + {%- endfor %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- message.content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} +{%- endif %} diff --git a/checkpoint-300/chat_template.jinja b/checkpoint-300/chat_template.jinja new file mode 100644 index 0000000..bdf7919 --- /dev/null +++ b/checkpoint-300/chat_template.jinja @@ -0,0 +1,54 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0]['role'] == 'system' %} + {{- messages[0]['content'] }} + {%- else %} + {{- 'You are Qwen, created by Alibaba Cloud. You are a helpful assistant.' }} + {%- endif %} + {{- "\n\n# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0]['role'] == 'system' %} + {{- '<|im_start|>system\n' + messages[0]['content'] + '<|im_end|>\n' }} + {%- else %} + {{- '<|im_start|>system\nYou are Qwen, created by Alibaba Cloud. You are a helpful assistant.<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- for message in messages %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) or (message.role == "assistant" and not message.tool_calls) %} + {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {{- '<|im_start|>' + message.role }} + {%- if message.content %} + {{- '\n' + message.content }} + {%- endif %} + {%- for tool_call in message.tool_calls %} + {%- if tool_call.function is defined %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {{- tool_call.arguments | tojson }} + {{- '}\n' }} + {%- endfor %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- message.content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} +{%- endif %} diff --git a/checkpoint-300/config.json b/checkpoint-300/config.json new file mode 100644 index 0000000..02328e4 --- /dev/null +++ b/checkpoint-300/config.json @@ -0,0 +1,57 @@ +{ + "architectures": [ + "Qwen2ForCausalLM" + ], + "attention_dropout": 0.0, + "bos_token_id": null, + "dtype": "float32", + "eos_token_id": 151645, + "hidden_act": "silu", + "hidden_size": 896, + "initializer_range": 0.02, + "intermediate_size": 4864, + "layer_types": [ + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention" + ], + "max_position_embeddings": 32768, + "max_window_layers": 24, + "model_type": "qwen2", + "num_attention_heads": 14, + "num_hidden_layers": 24, + "num_key_value_heads": 2, + "pad_token_id": 151643, + "rms_norm_eps": 1e-06, + "rope_parameters": { + "rope_theta": 1000000.0, + "rope_type": "default" + }, + "sliding_window": null, + "tie_word_embeddings": true, + "transformers_version": "5.6.2", + "use_cache": false, + "use_sliding_window": false, + "vocab_size": 151936 +} diff --git a/checkpoint-300/generation_config.json b/checkpoint-300/generation_config.json new file mode 100644 index 0000000..84f076a --- /dev/null +++ b/checkpoint-300/generation_config.json @@ -0,0 +1,13 @@ +{ + "do_sample": true, + "eos_token_id": [ + 151645, + 151643 + ], + "pad_token_id": 151643, + "repetition_penalty": 1.05, + "temperature": 0.7, + "top_k": 20, + "top_p": 0.8, + "transformers_version": "5.6.2" +} diff --git a/checkpoint-300/model.safetensors b/checkpoint-300/model.safetensors new file mode 100644 index 0000000..1dc9cfb --- /dev/null +++ b/checkpoint-300/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:275ebac6d53742a54d972d5a2fdf93a64ab774cf50af3c817e02a1376655c840 +size 1976163472 diff --git a/checkpoint-300/optimizer.pt b/checkpoint-300/optimizer.pt new file mode 100644 index 0000000..334ee45 --- /dev/null +++ b/checkpoint-300/optimizer.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3577491d619a3e9b2d76cba84e6eee9cdffd5bb2784ebc6a1e3453f2ce9f8021 +size 3952505274 diff --git a/checkpoint-300/rng_state.pth b/checkpoint-300/rng_state.pth new file mode 100644 index 0000000..0bc0aa2 --- /dev/null +++ b/checkpoint-300/rng_state.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fa9a1789e81962b242729edabc19959b88ccde1eb3dfdbc7cd826e14f85a76f9 +size 14244 diff --git a/checkpoint-300/scheduler.pt b/checkpoint-300/scheduler.pt new file mode 100644 index 0000000..863d8fc --- /dev/null +++ b/checkpoint-300/scheduler.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:262427cf509faa4beebbf93a0c170cf18cb00c5f988d14d843ea44ed3b3c2cae +size 1064 diff --git a/checkpoint-300/tokenizer.json b/checkpoint-300/tokenizer.json new file mode 100644 index 0000000..34510ff --- /dev/null +++ b/checkpoint-300/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3fd169731d2cbde95e10bf356d66d5997fd885dd8dbb6fb4684da3f23b2585d8 +size 11421892 diff --git a/checkpoint-300/tokenizer_config.json b/checkpoint-300/tokenizer_config.json new file mode 100644 index 0000000..f58460d --- /dev/null +++ b/checkpoint-300/tokenizer_config.json @@ -0,0 +1,32 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|im_end|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": false, + "local_files_only": false, + "model_max_length": 32768, + "pad_token": "<|endoftext|>", + "padding_side": "left", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "truncation_side": "left", + "unk_token": null +} diff --git a/checkpoint-300/trainer_state.json b/checkpoint-300/trainer_state.json new file mode 100644 index 0000000..8f432f9 --- /dev/null +++ b/checkpoint-300/trainer_state.json @@ -0,0 +1,1803 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 1.8404907975460123, + "eval_steps": 500, + "global_step": 300, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.75, + "completions/max_length": 512.0, + "completions/max_terminated_length": 37.0, + "completions/mean_length": 393.25, + "completions/mean_terminated_length": 37.0, + "completions/min_length": 37.0, + "completions/min_terminated_length": 37.0, + "entropy": 1.4897738695144653, + "epoch": 0.006134969325153374, + "frac_reward_zero_std": 0.5, + "grad_norm": 2.2988293170928955, + "learning_rate": 5e-07, + "loss": -0.21252349019050598, + "num_tokens": 3567.0, + "reward": -0.3424999713897705, + "reward_std": 0.01500000525265932, + "rewards/format_reward/mean": 0.02500000037252903, + "rewards/format_reward/std": 0.05000000074505806, + "rewards/security_audit_reward/mean": -0.5, + "rewards/security_audit_reward/std": 0.0, + "step": 1, + "step_time": 39.508622552999896 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.5, + "completions/max_length": 512.0, + "completions/max_terminated_length": 248.75, + "completions/mean_length": 389.625, + "completions/mean_terminated_length": 192.79166793823242, + "completions/min_length": 272.75, + "completions/min_terminated_length": 144.75, + "entropy": 1.363443061709404, + "epoch": 0.03067484662576687, + "frac_reward_zero_std": 0.375, + "grad_norm": 4.688082218170166, + "learning_rate": 4.938650306748465e-07, + "loss": 0.04808004945516586, + "num_tokens": 17675.0, + "reward": -0.2981249839067459, + "reward_std": 0.08178356755524874, + "rewards/format_reward/mean": 0.10000000381842256, + "rewards/format_reward/std": 0.12774468399584293, + "rewards/security_audit_reward/mean": -0.46875, + "rewards/security_audit_reward/std": 0.0625, + "step": 5, + "step_time": 38.500043476749966 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.65, + "completions/max_length": 512.0, + "completions/max_terminated_length": 345.6, + "completions/mean_length": 463.0, + "completions/mean_terminated_length": 305.6, + "completions/min_length": 363.8, + "completions/min_terminated_length": 261.4, + "entropy": 1.4113845229148865, + "epoch": 0.06134969325153374, + "frac_reward_zero_std": 0.4, + "grad_norm": 3.245452880859375, + "learning_rate": 4.86196319018405e-07, + "loss": -0.00041331946849823, + "num_tokens": 37093.0, + "reward": -0.29424998760223386, + "reward_std": 0.08391451295465231, + "rewards/format_reward/mean": 0.12750000804662703, + "rewards/format_reward/std": 0.16304838731884957, + "rewards/security_audit_reward/mean": -0.475, + "rewards/security_audit_reward/std": 0.05, + "step": 10, + "step_time": 39.192330704800135 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.6, + "completions/max_length": 512.0, + "completions/max_terminated_length": 394.0, + "completions/mean_length": 455.5, + "completions/mean_terminated_length": 352.9, + "completions/min_length": 311.8, + "completions/min_terminated_length": 311.8, + "entropy": 1.179759132862091, + "epoch": 0.09202453987730061, + "frac_reward_zero_std": 0.7, + "grad_norm": 2.9624693393707275, + "learning_rate": 4.785276073619632e-07, + "loss": 0.03452911972999573, + "num_tokens": 55311.0, + "reward": -0.2887499898672104, + "reward_std": 0.09658594038337469, + "rewards/format_reward/mean": 0.0875, + "rewards/format_reward/std": 0.08947573080658913, + "rewards/security_audit_reward/mean": -0.45, + "rewards/security_audit_reward/std": 0.1, + "step": 15, + "step_time": 38.30515608799997 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.45, + "completions/max_length": 512.0, + "completions/max_terminated_length": 395.2, + "completions/mean_length": 416.9, + "completions/mean_terminated_length": 328.76666870117185, + "completions/min_length": 260.8, + "completions/min_terminated_length": 260.8, + "entropy": 1.298638153076172, + "epoch": 0.12269938650306748, + "frac_reward_zero_std": 0.2, + "grad_norm": 4.034470081329346, + "learning_rate": 4.7085889570552147e-07, + "loss": -0.008246073126792907, + "num_tokens": 72771.0, + "reward": -0.23124998807907104, + "reward_std": 0.16768747363239528, + "rewards/format_reward/mean": 0.19750000424683095, + "rewards/format_reward/std": 0.2057904489338398, + "rewards/security_audit_reward/mean": -0.4149999976158142, + "rewards/security_audit_reward/std": 0.16999999880790712, + "step": 20, + "step_time": 37.87772348239996 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.3, + "completions/max_length": 512.0, + "completions/max_terminated_length": 423.4, + "completions/mean_length": 382.1, + "completions/mean_terminated_length": 334.1000091552734, + "completions/min_length": 236.0, + "completions/min_terminated_length": 236.0, + "entropy": 1.317835807800293, + "epoch": 0.15337423312883436, + "frac_reward_zero_std": 0.3, + "grad_norm": 2.853423595428467, + "learning_rate": 4.631901840490797e-07, + "loss": -0.013739901781082153, + "num_tokens": 89889.0, + "reward": -0.2974999874830246, + "reward_std": 0.15671177953481674, + "rewards/format_reward/mean": 0.17500000596046447, + "rewards/format_reward/std": 0.18444484770298003, + "rewards/security_audit_reward/mean": -0.5, + "rewards/security_audit_reward/std": 0.15773502588272095, + "step": 25, + "step_time": 38.74009619139997 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.65, + "completions/max_length": 512.0, + "completions/max_terminated_length": 346.0, + "completions/mean_length": 463.35, + "completions/mean_terminated_length": 295.3, + "completions/min_length": 337.8, + "completions/min_terminated_length": 235.4, + "entropy": 1.1444598376750945, + "epoch": 0.18404907975460122, + "frac_reward_zero_std": 0.2, + "grad_norm": 3.64375901222229, + "learning_rate": 4.55521472392638e-07, + "loss": -0.03970654606819153, + "num_tokens": 108664.0, + "reward": -0.3184999763965607, + "reward_std": 0.04019503518939018, + "rewards/format_reward/mean": 0.10499999970197678, + "rewards/format_reward/std": 0.13398344144225122, + "rewards/security_audit_reward/mean": -0.5, + "rewards/security_audit_reward/std": 0.0, + "step": 30, + "step_time": 38.56538706479987 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.4, + "completions/max_length": 504.2, + "completions/max_terminated_length": 454.6, + "completions/mean_length": 421.25, + "completions/mean_terminated_length": 386.0, + "completions/min_length": 328.6, + "completions/min_terminated_length": 328.6, + "entropy": 1.3522289156913758, + "epoch": 0.2147239263803681, + "frac_reward_zero_std": 0.3, + "grad_norm": 3.4385552406311035, + "learning_rate": 4.4785276073619634e-07, + "loss": -0.06348788738250732, + "num_tokens": 126953.0, + "reward": -0.32824997901916503, + "reward_std": 0.03220053892582655, + "rewards/format_reward/mean": 0.07250000201165677, + "rewards/format_reward/std": 0.10733511671423912, + "rewards/security_audit_reward/mean": -0.5, + "rewards/security_audit_reward/std": 0.0, + "step": 35, + "step_time": 37.87626404739986 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.55, + "completions/max_length": 505.2, + "completions/max_terminated_length": 334.4, + "completions/mean_length": 414.9, + "completions/mean_terminated_length": 240.3, + "completions/min_length": 243.0, + "completions/min_terminated_length": 140.6, + "entropy": 1.230024951696396, + "epoch": 0.24539877300613497, + "frac_reward_zero_std": 0.3, + "grad_norm": 4.400479793548584, + "learning_rate": 4.401840490797546e-07, + "loss": 0.11927952766418456, + "num_tokens": 144785.0, + "reward": -0.2897499829530716, + "reward_std": 0.12973095811903476, + "rewards/format_reward/mean": 0.14250000044703484, + "rewards/format_reward/std": 0.14365934804081917, + "rewards/security_audit_reward/mean": -0.475, + "rewards/security_audit_reward/std": 0.13164966106414794, + "step": 40, + "step_time": 37.8069536416001 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.6, + "completions/max_length": 472.2, + "completions/max_terminated_length": 190.6, + "completions/mean_length": 412.7, + "completions/mean_terminated_length": 160.85, + "completions/min_length": 333.8, + "completions/min_terminated_length": 129.0, + "entropy": 1.2133947968482972, + "epoch": 0.27607361963190186, + "frac_reward_zero_std": 0.1, + "grad_norm": 4.325937271118164, + "learning_rate": 4.3251533742331285e-07, + "loss": 0.025146520137786864, + "num_tokens": 162443.0, + "reward": -0.1574999876320362, + "reward_std": 0.2636621415615082, + "rewards/format_reward/mean": 0.24500001072883607, + "rewards/format_reward/std": 0.23762110471725464, + "rewards/security_audit_reward/mean": -0.32999999523162843, + "rewards/security_audit_reward/std": 0.28574271202087403, + "step": 45, + "step_time": 34.90182834920015 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.55, + "completions/max_length": 512.0, + "completions/max_terminated_length": 308.6, + "completions/mean_length": 397.05, + "completions/mean_terminated_length": 259.2, + "completions/min_length": 203.0, + "completions/min_terminated_length": 203.0, + "entropy": 1.4294291973114013, + "epoch": 0.3067484662576687, + "frac_reward_zero_std": 0.4, + "grad_norm": 3.9505743980407715, + "learning_rate": 4.2484662576687116e-07, + "loss": -0.08058007955551147, + "num_tokens": 180200.0, + "reward": -0.29249998927116394, + "reward_std": 0.10127481501549482, + "rewards/format_reward/mean": 0.0750000026077032, + "rewards/format_reward/std": 0.1127780631184578, + "rewards/security_audit_reward/mean": -0.45, + "rewards/security_audit_reward/std": 0.1, + "step": 50, + "step_time": 38.750808009400046 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.5, + "completions/max_length": 512.0, + "completions/max_terminated_length": 389.8, + "completions/mean_length": 404.65, + "completions/mean_terminated_length": 297.43333740234374, + "completions/min_length": 192.8, + "completions/min_terminated_length": 192.8, + "entropy": 1.2564165532588958, + "epoch": 0.3374233128834356, + "frac_reward_zero_std": 0.3, + "grad_norm": 3.3762269020080566, + "learning_rate": 4.171779141104294e-07, + "loss": -0.030467823147773743, + "num_tokens": 198109.0, + "reward": -0.2542499825358391, + "reward_std": 0.07489922866225243, + "rewards/format_reward/mean": 0.20250000841915608, + "rewards/format_reward/std": 0.1368803471326828, + "rewards/security_audit_reward/mean": -0.45, + "rewards/security_audit_reward/std": 0.05773502588272095, + "step": 55, + "step_time": 38.411275500399825 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.35, + "completions/max_length": 496.2, + "completions/max_terminated_length": 448.8, + "completions/mean_length": 394.3, + "completions/mean_terminated_length": 358.6166687011719, + "completions/min_length": 285.4, + "completions/min_terminated_length": 285.4, + "entropy": 1.2620218694210052, + "epoch": 0.36809815950920244, + "frac_reward_zero_std": 0.2, + "grad_norm": 2.8227944374084473, + "learning_rate": 4.095092024539877e-07, + "loss": 0.039707571268081665, + "num_tokens": 215747.0, + "reward": -0.2729999750852585, + "reward_std": 0.13599938787519933, + "rewards/format_reward/mean": 0.14000000432133675, + "rewards/format_reward/std": 0.14343783408403396, + "rewards/security_audit_reward/mean": -0.45, + "rewards/security_audit_reward/std": 0.1393846869468689, + "step": 60, + "step_time": 37.50645367139987 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.35, + "completions/max_length": 484.8, + "completions/max_terminated_length": 334.8, + "completions/mean_length": 381.35, + "completions/mean_terminated_length": 255.98333740234375, + "completions/min_length": 275.6, + "completions/min_terminated_length": 173.2, + "entropy": 1.2798833012580872, + "epoch": 0.3987730061349693, + "frac_reward_zero_std": 0.2, + "grad_norm": 3.4819753170013428, + "learning_rate": 4.01840490797546e-07, + "loss": -0.06275686025619506, + "num_tokens": 233162.0, + "reward": -0.201749986410141, + "reward_std": 0.2016347900032997, + "rewards/format_reward/mean": 0.20250000804662704, + "rewards/format_reward/std": 0.22137173414230346, + "rewards/security_audit_reward/mean": -0.375, + "rewards/security_audit_reward/std": 0.20773502588272094, + "step": 65, + "step_time": 37.139902984000216 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.6, + "completions/max_length": 512.0, + "completions/max_terminated_length": 289.8, + "completions/mean_length": 408.45, + "completions/mean_terminated_length": 251.7, + "completions/min_length": 207.0, + "completions/min_terminated_length": 207.0, + "entropy": 1.2134525895118713, + "epoch": 0.4294478527607362, + "frac_reward_zero_std": 0.3, + "grad_norm": 4.019806861877441, + "learning_rate": 3.941717791411043e-07, + "loss": 0.08099154829978943, + "num_tokens": 251321.0, + "reward": -0.27599998414516447, + "reward_std": 0.08945702444761991, + "rewards/format_reward/mean": 0.1300000037997961, + "rewards/format_reward/std": 0.1645726040005684, + "rewards/security_audit_reward/mean": -0.45, + "rewards/security_audit_reward/std": 0.05773502588272095, + "step": 70, + "step_time": 38.02127088899997 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.5, + "completions/max_length": 459.8, + "completions/max_terminated_length": 206.2, + "completions/mean_length": 348.9, + "completions/mean_terminated_length": 176.9, + "completions/min_length": 148.6, + "completions/min_terminated_length": 148.6, + "entropy": 1.3179432690143584, + "epoch": 0.4601226993865031, + "frac_reward_zero_std": 0.0, + "grad_norm": 6.28598690032959, + "learning_rate": 3.8650306748466255e-07, + "loss": -0.11171818971633911, + "num_tokens": 267725.0, + "reward": -0.19474998638033866, + "reward_std": 0.17031802013516426, + "rewards/format_reward/mean": 0.23750000447034836, + "rewards/format_reward/std": 0.17114628925919534, + "rewards/security_audit_reward/mean": -0.3800000011920929, + "rewards/security_audit_reward/std": 0.180902099609375, + "step": 75, + "step_time": 34.450158203000136 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.5, + "completions/max_length": 512.0, + "completions/max_terminated_length": 332.8, + "completions/mean_length": 400.75, + "completions/mean_terminated_length": 289.06666870117186, + "completions/min_length": 251.0, + "completions/min_terminated_length": 251.0, + "entropy": 1.1514661133289337, + "epoch": 0.49079754601226994, + "frac_reward_zero_std": 0.4, + "grad_norm": 2.8479247093200684, + "learning_rate": 3.788343558282208e-07, + "loss": 0.03145935535430908, + "num_tokens": 285726.0, + "reward": -0.2569999933242798, + "reward_std": 0.15333212018013, + "rewards/format_reward/mean": 0.1350000023841858, + "rewards/format_reward/std": 0.18636635541915894, + "rewards/security_audit_reward/mean": -0.425, + "rewards/security_audit_reward/std": 0.15, + "step": 80, + "step_time": 38.869779922999626 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.5, + "completions/max_length": 512.0, + "completions/max_terminated_length": 321.2, + "completions/mean_length": 398.5, + "completions/mean_terminated_length": 253.0, + "completions/min_length": 183.0, + "completions/min_terminated_length": 183.0, + "entropy": 1.244500571489334, + "epoch": 0.5214723926380368, + "frac_reward_zero_std": 0.3, + "grad_norm": 2.146970272064209, + "learning_rate": 3.7116564417177916e-07, + "loss": 0.06171210408210755, + "num_tokens": 304148.0, + "reward": -0.22524999380111693, + "reward_std": 0.19191497713327407, + "rewards/format_reward/mean": 0.18250000327825547, + "rewards/format_reward/std": 0.19969657957553863, + "rewards/security_audit_reward/mean": -0.4, + "rewards/security_audit_reward/std": 0.2, + "step": 85, + "step_time": 39.297288996000134 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.55, + "completions/max_length": 512.0, + "completions/max_terminated_length": 360.0, + "completions/mean_length": 419.5, + "completions/mean_terminated_length": 325.6, + "completions/min_length": 291.2, + "completions/min_terminated_length": 291.2, + "entropy": 1.206581747531891, + "epoch": 0.5521472392638037, + "frac_reward_zero_std": 0.2, + "grad_norm": 2.1158456802368164, + "learning_rate": 3.634969325153374e-07, + "loss": -0.06664568185806274, + "num_tokens": 321680.0, + "reward": -0.23324998915195466, + "reward_std": 0.1919491995126009, + "rewards/format_reward/mean": 0.1675000049173832, + "rewards/format_reward/std": 0.19863576367497443, + "rewards/security_audit_reward/mean": -0.40499999523162844, + "rewards/security_audit_reward/std": 0.1899999976158142, + "step": 90, + "step_time": 38.49561594039933 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.4, + "completions/max_length": 469.2, + "completions/max_terminated_length": 364.0, + "completions/mean_length": 383.2, + "completions/mean_terminated_length": 291.8333374023438, + "completions/min_length": 217.6, + "completions/min_terminated_length": 217.6, + "entropy": 1.217250692844391, + "epoch": 0.5828220858895705, + "frac_reward_zero_std": 0.4, + "grad_norm": 4.098232269287109, + "learning_rate": 3.558282208588957e-07, + "loss": 0.05211906433105469, + "num_tokens": 339350.0, + "reward": -0.2119999945163727, + "reward_std": 0.19894140996038914, + "rewards/format_reward/mean": 0.1799999989569187, + "rewards/format_reward/std": 0.23350853994488716, + "rewards/security_audit_reward/mean": -0.37999999821186065, + "rewards/security_audit_reward/std": 0.1911805212497711, + "step": 95, + "step_time": 35.83687614579994 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.5, + "completions/max_length": 512.0, + "completions/max_terminated_length": 345.4, + "completions/mean_length": 377.45, + "completions/mean_terminated_length": 290.6333343505859, + "completions/min_length": 240.4, + "completions/min_terminated_length": 240.4, + "entropy": 1.2783292949199676, + "epoch": 0.6134969325153374, + "frac_reward_zero_std": 0.3, + "grad_norm": 2.361516237258911, + "learning_rate": 3.48159509202454e-07, + "loss": 0.06258203387260437, + "num_tokens": 356239.0, + "reward": -0.20649999231100083, + "reward_std": 0.18195689767599105, + "rewards/format_reward/mean": 0.24499999433755876, + "rewards/format_reward/std": 0.19310407042503358, + "rewards/security_audit_reward/mean": -0.4, + "rewards/security_audit_reward/std": 0.2, + "step": 100, + "step_time": 38.47846096040011 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.3, + "completions/max_length": 463.2, + "completions/max_terminated_length": 394.6, + "completions/mean_length": 335.8, + "completions/mean_terminated_length": 268.3, + "completions/min_length": 139.6, + "completions/min_terminated_length": 139.6, + "entropy": 1.2529696226119995, + "epoch": 0.6441717791411042, + "frac_reward_zero_std": 0.2, + "grad_norm": 3.356074094772339, + "learning_rate": 3.4049079754601224e-07, + "loss": 0.003340443968772888, + "num_tokens": 373237.0, + "reward": -0.2567499876022339, + "reward_std": 0.27417250275611876, + "rewards/format_reward/mean": 0.14750000461935997, + "rewards/format_reward/std": 0.19759280756115913, + "rewards/security_audit_reward/mean": -0.4299999952316284, + "rewards/security_audit_reward/std": 0.3186576545238495, + "step": 105, + "step_time": 35.43083410320014 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.45, + "completions/max_length": 512.0, + "completions/max_terminated_length": 372.8, + "completions/mean_length": 419.15, + "completions/mean_terminated_length": 319.2666748046875, + "completions/min_length": 266.8, + "completions/min_terminated_length": 266.8, + "entropy": 1.1685741186141967, + "epoch": 0.6748466257668712, + "frac_reward_zero_std": 0.2, + "grad_norm": 4.318619728088379, + "learning_rate": 3.3282208588957055e-07, + "loss": -0.026089027523994446, + "num_tokens": 391784.0, + "reward": -0.2662499874830246, + "reward_std": 0.07884115856140853, + "rewards/format_reward/mean": 0.16250000558793545, + "rewards/format_reward/std": 0.14070439487695693, + "rewards/security_audit_reward/mean": -0.45, + "rewards/security_audit_reward/std": 0.05773502588272095, + "step": 110, + "step_time": 38.89281254739999 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.3, + "completions/max_length": 468.0, + "completions/max_terminated_length": 337.6, + "completions/mean_length": 330.1, + "completions/mean_terminated_length": 240.4166687011719, + "completions/min_length": 160.8, + "completions/min_terminated_length": 160.8, + "entropy": 1.2954379856586455, + "epoch": 0.7055214723926381, + "frac_reward_zero_std": 0.2, + "grad_norm": 3.293928384780884, + "learning_rate": 3.251533742331288e-07, + "loss": 0.17276796102523803, + "num_tokens": 408446.0, + "reward": -0.22849999666213988, + "reward_std": 0.1390242099761963, + "rewards/format_reward/mean": 0.2300000011920929, + "rewards/format_reward/std": 0.23302415013313293, + "rewards/security_audit_reward/mean": -0.425, + "rewards/security_audit_reward/std": 0.10773502588272095, + "step": 115, + "step_time": 35.73082293679981 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.55, + "completions/max_length": 512.0, + "completions/max_terminated_length": 281.2, + "completions/mean_length": 400.5, + "completions/mean_terminated_length": 248.73333740234375, + "completions/min_length": 211.8, + "completions/min_terminated_length": 211.8, + "entropy": 1.2283548831939697, + "epoch": 0.7361963190184049, + "frac_reward_zero_std": 0.2, + "grad_norm": 2.632479190826416, + "learning_rate": 3.174846625766871e-07, + "loss": 0.05111231803894043, + "num_tokens": 426822.0, + "reward": -0.22074998915195465, + "reward_std": 0.15957241374999284, + "rewards/format_reward/mean": 0.1975000012665987, + "rewards/format_reward/std": 0.1847505249083042, + "rewards/security_audit_reward/mean": -0.4, + "rewards/security_audit_reward/std": 0.15773502588272095, + "step": 120, + "step_time": 39.14503100519996 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.5, + "completions/max_length": 512.0, + "completions/max_terminated_length": 340.2, + "completions/mean_length": 401.4, + "completions/mean_terminated_length": 236.03333740234376, + "completions/min_length": 245.2, + "completions/min_terminated_length": 142.8, + "entropy": 1.307636547088623, + "epoch": 0.7668711656441718, + "frac_reward_zero_std": 0.3, + "grad_norm": 5.566491603851318, + "learning_rate": 3.0981595092024537e-07, + "loss": 0.003215853124856949, + "num_tokens": 444322.0, + "reward": -0.11199999079108239, + "reward_std": 0.2506739288568497, + "rewards/format_reward/mean": 0.2449999988079071, + "rewards/format_reward/std": 0.20622505843639374, + "rewards/security_audit_reward/mean": -0.26500000059604645, + "rewards/security_audit_reward/std": 0.278915548324585, + "step": 125, + "step_time": 38.70229864360026 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.35, + "completions/max_length": 512.0, + "completions/max_terminated_length": 390.6, + "completions/mean_length": 367.0, + "completions/mean_terminated_length": 264.0000061035156, + "completions/min_length": 123.2, + "completions/min_terminated_length": 123.2, + "entropy": 1.248900693655014, + "epoch": 0.7975460122699386, + "frac_reward_zero_std": 0.2, + "grad_norm": 4.395384311676025, + "learning_rate": 3.021472392638036e-07, + "loss": 0.06482647061347961, + "num_tokens": 461894.0, + "reward": -0.2042499899864197, + "reward_std": 0.17003463432192803, + "rewards/format_reward/mean": 0.2525000125169754, + "rewards/format_reward/std": 0.21560870110988617, + "rewards/security_audit_reward/mean": -0.4, + "rewards/security_audit_reward/std": 0.15773502588272095, + "step": 130, + "step_time": 39.174271353800215 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.35, + "completions/max_length": 512.0, + "completions/max_terminated_length": 370.0, + "completions/mean_length": 359.95, + "completions/mean_terminated_length": 276.9666687011719, + "completions/min_length": 203.8, + "completions/min_terminated_length": 203.8, + "entropy": 1.3299469709396363, + "epoch": 0.8282208588957055, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.3037519454956055, + "learning_rate": 2.94478527607362e-07, + "loss": 0.026153716444969177, + "num_tokens": 478783.0, + "reward": -0.19949999153614045, + "reward_std": 0.15296672135591508, + "rewards/format_reward/mean": 0.24500000178813935, + "rewards/format_reward/std": 0.2273508906364441, + "rewards/security_audit_reward/mean": -0.39000000059604645, + "rewards/security_audit_reward/std": 0.12891554832458496, + "step": 135, + "step_time": 38.658669441000164 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.45, + "completions/max_length": 512.0, + "completions/max_terminated_length": 354.0, + "completions/mean_length": 380.3, + "completions/mean_terminated_length": 271.6666687011719, + "completions/min_length": 206.4, + "completions/min_terminated_length": 206.4, + "entropy": 1.0997539341449738, + "epoch": 0.8588957055214724, + "frac_reward_zero_std": 0.2, + "grad_norm": 2.3693976402282715, + "learning_rate": 2.8680981595092024e-07, + "loss": -0.01876506209373474, + "num_tokens": 496243.0, + "reward": -0.17974998727440833, + "reward_std": 0.18710523881018162, + "rewards/format_reward/mean": 0.21750000193715097, + "rewards/format_reward/std": 0.1843859799206257, + "rewards/security_audit_reward/mean": -0.35, + "rewards/security_audit_reward/std": 0.20347774028778076, + "step": 140, + "step_time": 39.171343391999834 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.35, + "completions/max_length": 456.2, + "completions/max_terminated_length": 355.2, + "completions/mean_length": 348.35, + "completions/mean_terminated_length": 270.3, + "completions/min_length": 164.8, + "completions/min_terminated_length": 164.8, + "entropy": 1.2017314374446868, + "epoch": 0.8895705521472392, + "frac_reward_zero_std": 0.3, + "grad_norm": 4.230531215667725, + "learning_rate": 2.791411042944785e-07, + "loss": 0.0029310762882232668, + "num_tokens": 513422.0, + "reward": -0.1637499898672104, + "reward_std": 0.2463478922843933, + "rewards/format_reward/mean": 0.2475000023841858, + "rewards/format_reward/std": 0.2085829883813858, + "rewards/security_audit_reward/mean": -0.34000000059604646, + "rewards/security_audit_reward/std": 0.26830023527145386, + "step": 145, + "step_time": 34.95672115479992 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.25, + "completions/max_length": 482.4, + "completions/max_terminated_length": 431.6, + "completions/mean_length": 355.35, + "completions/mean_terminated_length": 319.4166687011719, + "completions/min_length": 212.0, + "completions/min_terminated_length": 212.0, + "entropy": 1.258862280845642, + "epoch": 0.9202453987730062, + "frac_reward_zero_std": 0.2, + "grad_norm": 6.071740627288818, + "learning_rate": 2.714723926380368e-07, + "loss": 0.0822126567363739, + "num_tokens": 530643.0, + "reward": -0.1912499874830246, + "reward_std": 0.1670845106244087, + "rewards/format_reward/mean": 0.27249999940395353, + "rewards/format_reward/std": 0.1733592666685581, + "rewards/security_audit_reward/mean": -0.39000000059604645, + "rewards/security_audit_reward/std": 0.17118052244186402, + "step": 150, + "step_time": 37.038782767599876 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.2, + "completions/max_length": 442.2, + "completions/max_terminated_length": 316.0, + "completions/mean_length": 283.75, + "completions/mean_terminated_length": 237.85000305175782, + "completions/min_length": 166.2, + "completions/min_terminated_length": 166.2, + "entropy": 1.4853489220142364, + "epoch": 0.950920245398773, + "frac_reward_zero_std": 0.0, + "grad_norm": 5.2570695877075195, + "learning_rate": 2.6380368098159506e-07, + "loss": 0.11004064083099366, + "num_tokens": 545966.0, + "reward": -0.15949999541044235, + "reward_std": 0.19192611873149873, + "rewards/format_reward/mean": 0.28500000238418577, + "rewards/format_reward/std": 0.22434256076812745, + "rewards/security_audit_reward/mean": -0.35, + "rewards/security_audit_reward/std": 0.2, + "step": 155, + "step_time": 33.86747411140077 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.55, + "completions/max_length": 512.0, + "completions/max_terminated_length": 340.8, + "completions/mean_length": 411.45, + "completions/mean_terminated_length": 265.9666687011719, + "completions/min_length": 192.0, + "completions/min_terminated_length": 192.0, + "entropy": 1.0828768193721772, + "epoch": 0.9815950920245399, + "frac_reward_zero_std": 0.2, + "grad_norm": 2.6537587642669678, + "learning_rate": 2.5613496932515337e-07, + "loss": 0.03556116819381714, + "num_tokens": 563683.0, + "reward": -0.20099999010562897, + "reward_std": 0.1888158166781068, + "rewards/format_reward/mean": 0.24000000059604645, + "rewards/format_reward/std": 0.16870398968458175, + "rewards/security_audit_reward/mean": -0.3899999976158142, + "rewards/security_audit_reward/std": 0.2199999988079071, + "step": 160, + "step_time": 38.665304075799575 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.3, + "completions/max_length": 483.2, + "completions/max_terminated_length": 393.2, + "completions/mean_length": 359.2, + "completions/mean_terminated_length": 303.3666687011719, + "completions/min_length": 178.2, + "completions/min_terminated_length": 178.2, + "entropy": 1.1811485469341279, + "epoch": 1.0122699386503067, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.203860282897949, + "learning_rate": 2.4846625766871163e-07, + "loss": -0.02532302737236023, + "num_tokens": 580183.0, + "reward": -0.1227499857544899, + "reward_std": 0.2651766210794449, + "rewards/format_reward/mean": 0.2675000011920929, + "rewards/format_reward/std": 0.24115291833877564, + "rewards/security_audit_reward/mean": -0.2899999976158142, + "rewards/security_audit_reward/std": 0.2812127649784088, + "step": 165, + "step_time": 36.34888075860035 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.4, + "completions/max_length": 512.0, + "completions/max_terminated_length": 345.8, + "completions/mean_length": 358.5, + "completions/mean_terminated_length": 235.96666870117187, + "completions/min_length": 123.0, + "completions/min_terminated_length": 123.0, + "entropy": 1.2863860994577407, + "epoch": 1.0429447852760736, + "frac_reward_zero_std": 0.2, + "grad_norm": 2.71185302734375, + "learning_rate": 2.4079754601226994e-07, + "loss": 0.12254136800765991, + "num_tokens": 597345.0, + "reward": -0.20274999886751174, + "reward_std": 0.1825057201087475, + "rewards/format_reward/mean": 0.25750000327825545, + "rewards/format_reward/std": 0.19183385372161865, + "rewards/security_audit_reward/mean": -0.4, + "rewards/security_audit_reward/std": 0.19711971282958984, + "step": 170, + "step_time": 38.96664929399922 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.5, + "completions/max_length": 512.0, + "completions/max_terminated_length": 323.4, + "completions/mean_length": 407.1, + "completions/mean_terminated_length": 234.40000610351564, + "completions/min_length": 242.8, + "completions/min_terminated_length": 140.4, + "entropy": 1.179810070991516, + "epoch": 1.0736196319018405, + "frac_reward_zero_std": 0.2, + "grad_norm": 4.413055419921875, + "learning_rate": 2.331288343558282e-07, + "loss": 0.041037318110466, + "num_tokens": 615063.0, + "reward": -0.20374999046325684, + "reward_std": 0.2052689865231514, + "rewards/format_reward/mean": 0.31250000894069674, + "rewards/format_reward/std": 0.22553626000881194, + "rewards/security_audit_reward/mean": -0.425, + "rewards/security_audit_reward/std": 0.23164966106414794, + "step": 175, + "step_time": 39.00492364500023 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.3, + "completions/max_length": 511.6, + "completions/max_terminated_length": 424.0, + "completions/mean_length": 399.45, + "completions/mean_terminated_length": 347.9166748046875, + "completions/min_length": 262.6, + "completions/min_terminated_length": 262.6, + "entropy": 1.1026120364665986, + "epoch": 1.1042944785276074, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.408414840698242, + "learning_rate": 2.254601226993865e-07, + "loss": 0.058766734600067136, + "num_tokens": 632696.0, + "reward": -0.16599998623132706, + "reward_std": 0.26377752125263215, + "rewards/format_reward/mean": 0.24000000655651094, + "rewards/format_reward/std": 0.21778101623058319, + "rewards/security_audit_reward/mean": -0.34000000059604646, + "rewards/security_audit_reward/std": 0.3105652093887329, + "step": 180, + "step_time": 39.09616019519963 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.2, + "completions/max_length": 477.4, + "completions/max_terminated_length": 421.6, + "completions/mean_length": 322.45, + "completions/mean_terminated_length": 273.1000030517578, + "completions/min_length": 156.0, + "completions/min_terminated_length": 156.0, + "entropy": 1.2888785600662231, + "epoch": 1.1349693251533743, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.6877431869506836, + "learning_rate": 2.1779141104294476e-07, + "loss": -0.0771723210811615, + "num_tokens": 649353.0, + "reward": -0.1799999952316284, + "reward_std": 0.3134476348757744, + "rewards/format_reward/mean": 0.2750000089406967, + "rewards/format_reward/std": 0.24135999679565429, + "rewards/security_audit_reward/mean": -0.375, + "rewards/security_audit_reward/std": 0.3593961834907532, + "step": 185, + "step_time": 36.71157897000012 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.3, + "completions/max_length": 495.0, + "completions/max_terminated_length": 351.0, + "completions/mean_length": 328.8, + "completions/mean_terminated_length": 243.7, + "completions/min_length": 164.8, + "completions/min_terminated_length": 164.8, + "entropy": 1.4585140287876128, + "epoch": 1.165644171779141, + "frac_reward_zero_std": 0.2, + "grad_norm": 4.987306118011475, + "learning_rate": 2.1012269938650307e-07, + "loss": -0.15080010890960693, + "num_tokens": 665513.0, + "reward": -0.050499990582466125, + "reward_std": 0.2684710592031479, + "rewards/format_reward/mean": 0.31000000387430193, + "rewards/format_reward/std": 0.1994625985622406, + "rewards/security_audit_reward/mean": -0.20500000119209288, + "rewards/security_audit_reward/std": 0.31255176067352297, + "step": 190, + "step_time": 37.63543628939988 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.5, + "completions/max_length": 512.0, + "completions/max_terminated_length": 307.8, + "completions/mean_length": 391.75, + "completions/mean_terminated_length": 253.06666870117186, + "completions/min_length": 178.0, + "completions/min_terminated_length": 178.0, + "entropy": 1.1293343544006347, + "epoch": 1.196319018404908, + "frac_reward_zero_std": 0.1, + "grad_norm": 5.247244358062744, + "learning_rate": 2.0245398773006135e-07, + "loss": -0.04229157567024231, + "num_tokens": 683268.0, + "reward": -0.10224998965859414, + "reward_std": 0.19266743455082178, + "rewards/format_reward/mean": 0.3124999929219484, + "rewards/format_reward/std": 0.1390557773411274, + "rewards/security_audit_reward/mean": -0.2800000011920929, + "rewards/security_audit_reward/std": 0.23863712549209595, + "step": 195, + "step_time": 38.93936442300037 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.4, + "completions/max_length": 485.0, + "completions/max_terminated_length": 346.2, + "completions/mean_length": 364.6, + "completions/mean_terminated_length": 265.8166748046875, + "completions/min_length": 165.8, + "completions/min_terminated_length": 165.8, + "entropy": 0.8287177711725235, + "epoch": 1.2269938650306749, + "frac_reward_zero_std": 0.1, + "grad_norm": 2.4230945110321045, + "learning_rate": 1.9478527607361963e-07, + "loss": -0.05633368492126465, + "num_tokens": 700760.0, + "reward": -0.1807499848306179, + "reward_std": 0.18529897555708885, + "rewards/format_reward/mean": 0.3075000137090683, + "rewards/format_reward/std": 0.15467575192451477, + "rewards/security_audit_reward/mean": -0.39000000059604645, + "rewards/security_audit_reward/std": 0.2105652093887329, + "step": 200, + "step_time": 37.065423558799736 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.25, + "completions/max_length": 431.6, + "completions/max_terminated_length": 338.8, + "completions/mean_length": 284.1, + "completions/mean_terminated_length": 225.65, + "completions/min_length": 119.4, + "completions/min_terminated_length": 119.4, + "entropy": 1.2736368715763091, + "epoch": 1.2576687116564418, + "frac_reward_zero_std": 0.1, + "grad_norm": 4.797567367553711, + "learning_rate": 1.8711656441717791e-07, + "loss": 0.08297693133354186, + "num_tokens": 716344.0, + "reward": -0.07274999544024467, + "reward_std": 0.24350565671920776, + "rewards/format_reward/mean": 0.31749999821186065, + "rewards/format_reward/std": 0.19669782146811485, + "rewards/security_audit_reward/mean": -0.23999999985098838, + "rewards/security_audit_reward/std": 0.2692204549908638, + "step": 205, + "step_time": 33.11877055760014 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.25, + "completions/max_length": 484.0, + "completions/max_terminated_length": 403.2, + "completions/mean_length": 335.45, + "completions/mean_terminated_length": 276.4500030517578, + "completions/min_length": 173.8, + "completions/min_terminated_length": 173.8, + "entropy": 1.1084223449230195, + "epoch": 1.2883435582822087, + "frac_reward_zero_std": 0.1, + "grad_norm": 2.4603023529052734, + "learning_rate": 1.7944785276073617e-07, + "loss": 0.07945090532302856, + "num_tokens": 733245.0, + "reward": -0.13774999380111694, + "reward_std": 0.2730386942625046, + "rewards/format_reward/mean": 0.2174999989569187, + "rewards/format_reward/std": 0.22229814901947975, + "rewards/security_audit_reward/mean": -0.29000000059604647, + "rewards/security_audit_reward/std": 0.3105652093887329, + "step": 210, + "step_time": 37.03591289120122 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.4, + "completions/max_length": 512.0, + "completions/max_terminated_length": 361.8, + "completions/mean_length": 345.0, + "completions/mean_terminated_length": 261.3000030517578, + "completions/min_length": 184.8, + "completions/min_terminated_length": 184.8, + "entropy": 1.2274070978164673, + "epoch": 1.3190184049079754, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.573819637298584, + "learning_rate": 1.7177914110429448e-07, + "loss": -0.07497722506523133, + "num_tokens": 749917.0, + "reward": -0.01174999624490738, + "reward_std": 0.3002330154180527, + "rewards/format_reward/mean": 0.3225000023841858, + "rewards/format_reward/std": 0.1751384623348713, + "rewards/security_audit_reward/mean": -0.15499999821186067, + "rewards/security_audit_reward/std": 0.3648489773273468, + "step": 215, + "step_time": 38.89012140319937 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.2, + "completions/max_length": 481.8, + "completions/max_terminated_length": 344.6, + "completions/mean_length": 292.3, + "completions/mean_terminated_length": 234.26666870117188, + "completions/min_length": 122.2, + "completions/min_terminated_length": 122.2, + "entropy": 1.1711494624614716, + "epoch": 1.3496932515337423, + "frac_reward_zero_std": 0.1, + "grad_norm": 3.879939556121826, + "learning_rate": 1.6411042944785276e-07, + "loss": 0.06901218891143798, + "num_tokens": 765457.0, + "reward": -0.2002499908208847, + "reward_std": 0.19745510853827, + "rewards/format_reward/mean": 0.20749999657273294, + "rewards/format_reward/std": 0.2098293460905552, + "rewards/security_audit_reward/mean": -0.375, + "rewards/security_audit_reward/std": 0.20773502588272094, + "step": 220, + "step_time": 36.50884771559977 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.35, + "completions/max_length": 451.4, + "completions/max_terminated_length": 281.8, + "completions/mean_length": 310.85, + "completions/mean_terminated_length": 210.58333435058594, + "completions/min_length": 144.0, + "completions/min_terminated_length": 144.0, + "entropy": 1.4326449751853942, + "epoch": 1.3803680981595092, + "frac_reward_zero_std": 0.0, + "grad_norm": 5.483170986175537, + "learning_rate": 1.5644171779141104e-07, + "loss": -0.03490494191646576, + "num_tokens": 782226.0, + "reward": -0.14649999141693115, + "reward_std": 0.19791007936000823, + "rewards/format_reward/mean": 0.3049999952316284, + "rewards/format_reward/std": 0.19433450996875762, + "rewards/security_audit_reward/mean": -0.3399999998509884, + "rewards/security_audit_reward/std": 0.22000000029802322, + "step": 225, + "step_time": 35.079213985799655 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.35, + "completions/max_length": 511.2, + "completions/max_terminated_length": 315.2, + "completions/mean_length": 338.15, + "completions/mean_terminated_length": 226.28333435058593, + "completions/min_length": 134.8, + "completions/min_terminated_length": 134.8, + "entropy": 1.1364098012447357, + "epoch": 1.4110429447852761, + "frac_reward_zero_std": 0.2, + "grad_norm": 3.3578364849090576, + "learning_rate": 1.4877300613496933e-07, + "loss": 0.0896155834197998, + "num_tokens": 798571.0, + "reward": -0.11574998870491982, + "reward_std": 0.19651760943233967, + "rewards/format_reward/mean": 0.2674999989569187, + "rewards/format_reward/std": 0.15046989992260934, + "rewards/security_audit_reward/mean": -0.27999999821186067, + "rewards/security_audit_reward/std": 0.2297215759754181, + "step": 230, + "step_time": 38.61798697480081 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.35, + "completions/max_length": 505.4, + "completions/max_terminated_length": 385.4, + "completions/mean_length": 360.65, + "completions/mean_terminated_length": 295.23333740234375, + "completions/min_length": 210.2, + "completions/min_terminated_length": 210.2, + "entropy": 1.0565216183662414, + "epoch": 1.441717791411043, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.266097068786621, + "learning_rate": 1.4110429447852758e-07, + "loss": 0.07759050726890564, + "num_tokens": 815570.0, + "reward": -0.06749999299645423, + "reward_std": 0.27374918162822726, + "rewards/format_reward/mean": 0.37000001072883604, + "rewards/format_reward/std": 0.1865294199436903, + "rewards/security_audit_reward/mean": -0.2550000011920929, + "rewards/security_audit_reward/std": 0.32802181243896483, + "step": 235, + "step_time": 38.44030983600023 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.3, + "completions/max_length": 495.6, + "completions/max_terminated_length": 352.6, + "completions/mean_length": 336.05, + "completions/mean_terminated_length": 243.60000915527343, + "completions/min_length": 145.8, + "completions/min_terminated_length": 145.8, + "entropy": 1.417020809650421, + "epoch": 1.4723926380368098, + "frac_reward_zero_std": 0.2, + "grad_norm": 4.767548084259033, + "learning_rate": 1.334355828220859e-07, + "loss": 0.03671485185623169, + "num_tokens": 831713.0, + "reward": -0.12849999219179153, + "reward_std": 0.19381159394979477, + "rewards/format_reward/mean": 0.3300000041723251, + "rewards/format_reward/std": 0.20168980173766612, + "rewards/security_audit_reward/mean": -0.325, + "rewards/security_audit_reward/std": 0.20773502588272094, + "step": 240, + "step_time": 37.49174958900003 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.6, + "completions/max_length": 512.0, + "completions/max_terminated_length": 343.0, + "completions/mean_length": 417.3, + "completions/mean_terminated_length": 293.7, + "completions/min_length": 246.8, + "completions/min_terminated_length": 246.8, + "entropy": 1.0088598132133484, + "epoch": 1.5030674846625767, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.1936607360839844, + "learning_rate": 1.2576687116564417e-07, + "loss": -0.03240810632705689, + "num_tokens": 849855.0, + "reward": -0.11324999332427979, + "reward_std": 0.2602782666683197, + "rewards/format_reward/mean": 0.3224999994039536, + "rewards/format_reward/std": 0.20757876634597777, + "rewards/security_audit_reward/mean": -0.3, + "rewards/security_audit_reward/std": 0.3154700517654419, + "step": 245, + "step_time": 39.07302968719996 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.2, + "completions/max_length": 459.6, + "completions/max_terminated_length": 428.2, + "completions/mean_length": 325.35, + "completions/mean_terminated_length": 277.5833374023438, + "completions/min_length": 137.0, + "completions/min_terminated_length": 137.0, + "entropy": 1.1867628961801528, + "epoch": 1.5337423312883436, + "frac_reward_zero_std": 0.1, + "grad_norm": 3.9449942111968994, + "learning_rate": 1.1809815950920244e-07, + "loss": -0.005416367202997208, + "num_tokens": 866330.0, + "reward": -0.08199999034404755, + "reward_std": 0.2747137784957886, + "rewards/format_reward/mean": 0.3099999874830246, + "rewards/format_reward/std": 0.17935641929507257, + "rewards/security_audit_reward/mean": -0.25, + "rewards/security_audit_reward/std": 0.33094010353088377, + "step": 250, + "step_time": 35.27377968320034 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.25, + "completions/max_length": 450.0, + "completions/max_terminated_length": 311.2, + "completions/mean_length": 283.8, + "completions/mean_terminated_length": 202.06666870117186, + "completions/min_length": 128.0, + "completions/min_terminated_length": 128.0, + "entropy": 1.322118791937828, + "epoch": 1.5644171779141103, + "frac_reward_zero_std": 0.1, + "grad_norm": 6.9948039054870605, + "learning_rate": 1.1042944785276073e-07, + "loss": 0.055895209312438965, + "num_tokens": 881650.0, + "reward": -0.11924999132752419, + "reward_std": 0.14510822538286447, + "rewards/format_reward/mean": 0.3024999976158142, + "rewards/format_reward/std": 0.15741010159254074, + "rewards/security_audit_reward/mean": -0.3, + "rewards/security_audit_reward/std": 0.15773502588272095, + "step": 255, + "step_time": 34.332786842800125 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.3, + "completions/max_length": 485.0, + "completions/max_terminated_length": 312.2, + "completions/mean_length": 312.0, + "completions/mean_terminated_length": 225.13333740234376, + "completions/min_length": 141.8, + "completions/min_terminated_length": 141.8, + "entropy": 1.1261488378047944, + "epoch": 1.5950920245398774, + "frac_reward_zero_std": 0.1, + "grad_norm": 5.633542537689209, + "learning_rate": 1.0276073619631902e-07, + "loss": 0.12422184944152832, + "num_tokens": 898170.0, + "reward": -0.09424999356269836, + "reward_std": 0.2452640563249588, + "rewards/format_reward/mean": 0.3274999916553497, + "rewards/format_reward/std": 0.19424656331539153, + "rewards/security_audit_reward/mean": -0.275, + "rewards/security_audit_reward/std": 0.29574271440505984, + "step": 260, + "step_time": 37.18088837539908 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.3, + "completions/max_length": 481.4, + "completions/max_terminated_length": 333.8, + "completions/mean_length": 323.4, + "completions/mean_terminated_length": 233.95, + "completions/min_length": 174.6, + "completions/min_terminated_length": 174.6, + "entropy": 1.3164357602596284, + "epoch": 1.6257668711656441, + "frac_reward_zero_std": 0.2, + "grad_norm": 4.177126884460449, + "learning_rate": 9.50920245398773e-08, + "loss": -0.0031075358390808107, + "num_tokens": 914438.0, + "reward": -0.109499990940094, + "reward_std": 0.19355954378843307, + "rewards/format_reward/mean": 0.3700000077486038, + "rewards/format_reward/std": 0.19431518614292145, + "rewards/security_audit_reward/mean": -0.31500000059604644, + "rewards/security_audit_reward/std": 0.21745660305023193, + "step": 265, + "step_time": 36.75480746599969 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.15, + "completions/max_length": 460.2, + "completions/max_terminated_length": 279.2, + "completions/mean_length": 224.9, + "completions/mean_terminated_length": 170.45000305175782, + "completions/min_length": 84.4, + "completions/min_terminated_length": 84.4, + "entropy": 1.267154586315155, + "epoch": 1.656441717791411, + "frac_reward_zero_std": 0.2, + "grad_norm": 7.552680015563965, + "learning_rate": 8.742331288343557e-08, + "loss": -0.12154214382171631, + "num_tokens": 928536.0, + "reward": -0.07999998778104782, + "reward_std": 0.16925212144851684, + "rewards/format_reward/mean": 0.37499999403953554, + "rewards/format_reward/std": 0.1521439790725708, + "rewards/security_audit_reward/mean": -0.275, + "rewards/security_audit_reward/std": 0.20773502588272094, + "step": 270, + "step_time": 35.04094967719975 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.3, + "completions/max_length": 469.4, + "completions/max_terminated_length": 287.6, + "completions/mean_length": 300.95, + "completions/mean_terminated_length": 204.76667175292968, + "completions/min_length": 130.0, + "completions/min_terminated_length": 130.0, + "entropy": 1.1936017721891403, + "epoch": 1.687116564417178, + "frac_reward_zero_std": 0.2, + "grad_norm": 7.106525421142578, + "learning_rate": 7.975460122699386e-08, + "loss": -0.0458857923746109, + "num_tokens": 944307.0, + "reward": -0.05174999088048935, + "reward_std": 0.23333178758621215, + "rewards/format_reward/mean": 0.3874999940395355, + "rewards/format_reward/std": 0.16559004038572311, + "rewards/security_audit_reward/mean": -0.24000000059604645, + "rewards/security_audit_reward/std": 0.2866505742073059, + "step": 275, + "step_time": 36.0470411268001 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.2, + "completions/max_length": 498.6, + "completions/max_terminated_length": 375.0, + "completions/mean_length": 295.25, + "completions/mean_terminated_length": 232.6666717529297, + "completions/min_length": 97.8, + "completions/min_terminated_length": 97.8, + "entropy": 1.460896384716034, + "epoch": 1.7177914110429446, + "frac_reward_zero_std": 0.0, + "grad_norm": 5.004129409790039, + "learning_rate": 7.208588957055214e-08, + "loss": -0.10658804178237916, + "num_tokens": 960078.0, + "reward": -0.01824999153614044, + "reward_std": 0.2521414369344711, + "rewards/format_reward/mean": 0.3825000047683716, + "rewards/format_reward/std": 0.15327396541833876, + "rewards/security_audit_reward/mean": -0.1899999976158142, + "rewards/security_audit_reward/std": 0.3234777390956879, + "step": 280, + "step_time": 37.26412723539943 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.3, + "completions/max_length": 501.0, + "completions/max_terminated_length": 363.2, + "completions/mean_length": 340.6, + "completions/mean_terminated_length": 257.06666870117186, + "completions/min_length": 146.2, + "completions/min_terminated_length": 146.2, + "entropy": 1.0431257128715514, + "epoch": 1.7484662576687118, + "frac_reward_zero_std": 0.1, + "grad_norm": 4.059199333190918, + "learning_rate": 6.441717791411043e-08, + "loss": -0.10386581420898437, + "num_tokens": 976992.0, + "reward": -0.06924999207258224, + "reward_std": 0.2484972782433033, + "rewards/format_reward/mean": 0.38750000596046447, + "rewards/format_reward/std": 0.1250488668680191, + "rewards/security_audit_reward/mean": -0.26500000059604645, + "rewards/security_audit_reward/std": 0.3080150008201599, + "step": 285, + "step_time": 38.17630281240017 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.05, + "completions/max_length": 435.0, + "completions/max_terminated_length": 413.6, + "completions/mean_length": 281.65, + "completions/mean_terminated_length": 274.0, + "completions/min_length": 152.0, + "completions/min_terminated_length": 152.0, + "entropy": 1.1775987446308136, + "epoch": 1.7791411042944785, + "frac_reward_zero_std": 0.3, + "grad_norm": 4.12350606918335, + "learning_rate": 5.674846625766871e-08, + "loss": -0.03856886327266693, + "num_tokens": 992819.0, + "reward": -0.05999999046325684, + "reward_std": 0.1456713281571865, + "rewards/format_reward/mean": 0.360000005364418, + "rewards/format_reward/std": 0.12044776938855647, + "rewards/security_audit_reward/mean": -0.23999999985098838, + "rewards/security_audit_reward/std": 0.17773502618074416, + "step": 290, + "step_time": 33.279480656001034 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.15, + "completions/max_length": 470.0, + "completions/max_terminated_length": 383.0, + "completions/mean_length": 306.15, + "completions/mean_terminated_length": 264.66666717529296, + "completions/min_length": 170.8, + "completions/min_terminated_length": 170.8, + "entropy": 1.3437508165836334, + "epoch": 1.8098159509202454, + "frac_reward_zero_std": 0.0, + "grad_norm": 5.014013290405273, + "learning_rate": 4.907975460122699e-08, + "loss": 0.1800641179084778, + "num_tokens": 1008676.0, + "reward": -0.14524998962879182, + "reward_std": 0.19439554661512376, + "rewards/format_reward/mean": 0.33249999284744264, + "rewards/format_reward/std": 0.21220951080322265, + "rewards/security_audit_reward/mean": -0.35, + "rewards/security_audit_reward/std": 0.2154700517654419, + "step": 295, + "step_time": 35.92737517459973 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.3, + "completions/max_length": 471.6, + "completions/max_terminated_length": 299.6, + "completions/mean_length": 360.45, + "completions/mean_terminated_length": 234.9166687011719, + "completions/min_length": 229.0, + "completions/min_terminated_length": 126.6, + "entropy": 1.1840724140405654, + "epoch": 1.8404907975460123, + "frac_reward_zero_std": 0.1, + "grad_norm": 2.570652484893799, + "learning_rate": 4.1411042944785274e-08, + "loss": 0.019638296961784363, + "num_tokens": 1025285.0, + "reward": -0.06699999049305916, + "reward_std": 0.2400740846991539, + "rewards/format_reward/mean": 0.3600000023841858, + "rewards/format_reward/std": 0.1531308189034462, + "rewards/security_audit_reward/mean": -0.25, + "rewards/security_audit_reward/std": 0.2868344783782959, + "step": 300, + "step_time": 35.41554451999982 + } + ], + "logging_steps": 5, + "max_steps": 326, + "num_input_tokens_seen": 1025285, + "num_train_epochs": 2, + "save_steps": 50, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 0.0, + "train_batch_size": 2, + "trial_name": null, + "trial_params": null +} diff --git a/checkpoint-300/training_args.bin b/checkpoint-300/training_args.bin new file mode 100644 index 0000000..93d9ca8 --- /dev/null +++ b/checkpoint-300/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b51f3856815802830b7add9e23ddd089207e5c9941078dd606f120af0f983d09 +size 6776 diff --git a/checkpoint-326/chat_template.jinja b/checkpoint-326/chat_template.jinja new file mode 100644 index 0000000..bdf7919 --- /dev/null +++ b/checkpoint-326/chat_template.jinja @@ -0,0 +1,54 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0]['role'] == 'system' %} + {{- messages[0]['content'] }} + {%- else %} + {{- 'You are Qwen, created by Alibaba Cloud. You are a helpful assistant.' }} + {%- endif %} + {{- "\n\n# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0]['role'] == 'system' %} + {{- '<|im_start|>system\n' + messages[0]['content'] + '<|im_end|>\n' }} + {%- else %} + {{- '<|im_start|>system\nYou are Qwen, created by Alibaba Cloud. You are a helpful assistant.<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- for message in messages %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) or (message.role == "assistant" and not message.tool_calls) %} + {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {{- '<|im_start|>' + message.role }} + {%- if message.content %} + {{- '\n' + message.content }} + {%- endif %} + {%- for tool_call in message.tool_calls %} + {%- if tool_call.function is defined %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {{- tool_call.arguments | tojson }} + {{- '}\n' }} + {%- endfor %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- message.content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} +{%- endif %} diff --git a/checkpoint-326/config.json b/checkpoint-326/config.json new file mode 100644 index 0000000..02328e4 --- /dev/null +++ b/checkpoint-326/config.json @@ -0,0 +1,57 @@ +{ + "architectures": [ + "Qwen2ForCausalLM" + ], + "attention_dropout": 0.0, + "bos_token_id": null, + "dtype": "float32", + "eos_token_id": 151645, + "hidden_act": "silu", + "hidden_size": 896, + "initializer_range": 0.02, + "intermediate_size": 4864, + "layer_types": [ + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention" + ], + "max_position_embeddings": 32768, + "max_window_layers": 24, + "model_type": "qwen2", + "num_attention_heads": 14, + "num_hidden_layers": 24, + "num_key_value_heads": 2, + "pad_token_id": 151643, + "rms_norm_eps": 1e-06, + "rope_parameters": { + "rope_theta": 1000000.0, + "rope_type": "default" + }, + "sliding_window": null, + "tie_word_embeddings": true, + "transformers_version": "5.6.2", + "use_cache": false, + "use_sliding_window": false, + "vocab_size": 151936 +} diff --git a/checkpoint-326/generation_config.json b/checkpoint-326/generation_config.json new file mode 100644 index 0000000..84f076a --- /dev/null +++ b/checkpoint-326/generation_config.json @@ -0,0 +1,13 @@ +{ + "do_sample": true, + "eos_token_id": [ + 151645, + 151643 + ], + "pad_token_id": 151643, + "repetition_penalty": 1.05, + "temperature": 0.7, + "top_k": 20, + "top_p": 0.8, + "transformers_version": "5.6.2" +} diff --git a/checkpoint-326/model.safetensors b/checkpoint-326/model.safetensors new file mode 100644 index 0000000..4df0c55 --- /dev/null +++ b/checkpoint-326/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3c64421ad1b2b08b8f84687a636657a68a9f6c9ef639c6c2dc449cb93d2c4219 +size 1976163472 diff --git a/checkpoint-326/optimizer.pt b/checkpoint-326/optimizer.pt new file mode 100644 index 0000000..770534b --- /dev/null +++ b/checkpoint-326/optimizer.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2029a2d6a0a790f550621622c3c990b0d5f81492de3d7642988e1fa042d3a073 +size 3952505274 diff --git a/checkpoint-326/rng_state.pth b/checkpoint-326/rng_state.pth new file mode 100644 index 0000000..33f910b --- /dev/null +++ b/checkpoint-326/rng_state.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7709cb910b037a9984c235d2fc7fe7fd99ccb8982993a4ca396269149709e777 +size 14244 diff --git a/checkpoint-326/scheduler.pt b/checkpoint-326/scheduler.pt new file mode 100644 index 0000000..3ad65e3 --- /dev/null +++ b/checkpoint-326/scheduler.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f3aa6a7bb4866149bd0f10ff54d9da7e2c37c93aa3f37a2a6471c11bd6760f19 +size 1064 diff --git a/checkpoint-326/tokenizer.json b/checkpoint-326/tokenizer.json new file mode 100644 index 0000000..34510ff --- /dev/null +++ b/checkpoint-326/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3fd169731d2cbde95e10bf356d66d5997fd885dd8dbb6fb4684da3f23b2585d8 +size 11421892 diff --git a/checkpoint-326/tokenizer_config.json b/checkpoint-326/tokenizer_config.json new file mode 100644 index 0000000..f58460d --- /dev/null +++ b/checkpoint-326/tokenizer_config.json @@ -0,0 +1,32 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|im_end|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": false, + "local_files_only": false, + "model_max_length": 32768, + "pad_token": "<|endoftext|>", + "padding_side": "left", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "truncation_side": "left", + "unk_token": null +} diff --git a/checkpoint-326/trainer_state.json b/checkpoint-326/trainer_state.json new file mode 100644 index 0000000..1360735 --- /dev/null +++ b/checkpoint-326/trainer_state.json @@ -0,0 +1,1948 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 2.0, + "eval_steps": 500, + "global_step": 326, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.75, + "completions/max_length": 512.0, + "completions/max_terminated_length": 37.0, + "completions/mean_length": 393.25, + "completions/mean_terminated_length": 37.0, + "completions/min_length": 37.0, + "completions/min_terminated_length": 37.0, + "entropy": 1.4897738695144653, + "epoch": 0.006134969325153374, + "frac_reward_zero_std": 0.5, + "grad_norm": 2.2988293170928955, + "learning_rate": 5e-07, + "loss": -0.21252349019050598, + "num_tokens": 3567.0, + "reward": -0.3424999713897705, + "reward_std": 0.01500000525265932, + "rewards/format_reward/mean": 0.02500000037252903, + "rewards/format_reward/std": 0.05000000074505806, + "rewards/security_audit_reward/mean": -0.5, + "rewards/security_audit_reward/std": 0.0, + "step": 1, + "step_time": 39.508622552999896 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.5, + "completions/max_length": 512.0, + "completions/max_terminated_length": 248.75, + "completions/mean_length": 389.625, + "completions/mean_terminated_length": 192.79166793823242, + "completions/min_length": 272.75, + "completions/min_terminated_length": 144.75, + "entropy": 1.363443061709404, + "epoch": 0.03067484662576687, + "frac_reward_zero_std": 0.375, + "grad_norm": 4.688082218170166, + "learning_rate": 4.938650306748465e-07, + "loss": 0.04808004945516586, + "num_tokens": 17675.0, + "reward": -0.2981249839067459, + "reward_std": 0.08178356755524874, + "rewards/format_reward/mean": 0.10000000381842256, + "rewards/format_reward/std": 0.12774468399584293, + "rewards/security_audit_reward/mean": -0.46875, + "rewards/security_audit_reward/std": 0.0625, + "step": 5, + "step_time": 38.500043476749966 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.65, + "completions/max_length": 512.0, + "completions/max_terminated_length": 345.6, + "completions/mean_length": 463.0, + "completions/mean_terminated_length": 305.6, + "completions/min_length": 363.8, + "completions/min_terminated_length": 261.4, + "entropy": 1.4113845229148865, + "epoch": 0.06134969325153374, + "frac_reward_zero_std": 0.4, + "grad_norm": 3.245452880859375, + "learning_rate": 4.86196319018405e-07, + "loss": -0.00041331946849823, + "num_tokens": 37093.0, + "reward": -0.29424998760223386, + "reward_std": 0.08391451295465231, + "rewards/format_reward/mean": 0.12750000804662703, + "rewards/format_reward/std": 0.16304838731884957, + "rewards/security_audit_reward/mean": -0.475, + "rewards/security_audit_reward/std": 0.05, + "step": 10, + "step_time": 39.192330704800135 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.6, + "completions/max_length": 512.0, + "completions/max_terminated_length": 394.0, + "completions/mean_length": 455.5, + "completions/mean_terminated_length": 352.9, + "completions/min_length": 311.8, + "completions/min_terminated_length": 311.8, + "entropy": 1.179759132862091, + "epoch": 0.09202453987730061, + "frac_reward_zero_std": 0.7, + "grad_norm": 2.9624693393707275, + "learning_rate": 4.785276073619632e-07, + "loss": 0.03452911972999573, + "num_tokens": 55311.0, + "reward": -0.2887499898672104, + "reward_std": 0.09658594038337469, + "rewards/format_reward/mean": 0.0875, + "rewards/format_reward/std": 0.08947573080658913, + "rewards/security_audit_reward/mean": -0.45, + "rewards/security_audit_reward/std": 0.1, + "step": 15, + "step_time": 38.30515608799997 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.45, + "completions/max_length": 512.0, + "completions/max_terminated_length": 395.2, + "completions/mean_length": 416.9, + "completions/mean_terminated_length": 328.76666870117185, + "completions/min_length": 260.8, + "completions/min_terminated_length": 260.8, + "entropy": 1.298638153076172, + "epoch": 0.12269938650306748, + "frac_reward_zero_std": 0.2, + "grad_norm": 4.034470081329346, + "learning_rate": 4.7085889570552147e-07, + "loss": -0.008246073126792907, + "num_tokens": 72771.0, + "reward": -0.23124998807907104, + "reward_std": 0.16768747363239528, + "rewards/format_reward/mean": 0.19750000424683095, + "rewards/format_reward/std": 0.2057904489338398, + "rewards/security_audit_reward/mean": -0.4149999976158142, + "rewards/security_audit_reward/std": 0.16999999880790712, + "step": 20, + "step_time": 37.87772348239996 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.3, + "completions/max_length": 512.0, + "completions/max_terminated_length": 423.4, + "completions/mean_length": 382.1, + "completions/mean_terminated_length": 334.1000091552734, + "completions/min_length": 236.0, + "completions/min_terminated_length": 236.0, + "entropy": 1.317835807800293, + "epoch": 0.15337423312883436, + "frac_reward_zero_std": 0.3, + "grad_norm": 2.853423595428467, + "learning_rate": 4.631901840490797e-07, + "loss": -0.013739901781082153, + "num_tokens": 89889.0, + "reward": -0.2974999874830246, + "reward_std": 0.15671177953481674, + "rewards/format_reward/mean": 0.17500000596046447, + "rewards/format_reward/std": 0.18444484770298003, + "rewards/security_audit_reward/mean": -0.5, + "rewards/security_audit_reward/std": 0.15773502588272095, + "step": 25, + "step_time": 38.74009619139997 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.65, + "completions/max_length": 512.0, + "completions/max_terminated_length": 346.0, + "completions/mean_length": 463.35, + "completions/mean_terminated_length": 295.3, + "completions/min_length": 337.8, + "completions/min_terminated_length": 235.4, + "entropy": 1.1444598376750945, + "epoch": 0.18404907975460122, + "frac_reward_zero_std": 0.2, + "grad_norm": 3.64375901222229, + "learning_rate": 4.55521472392638e-07, + "loss": -0.03970654606819153, + "num_tokens": 108664.0, + "reward": -0.3184999763965607, + "reward_std": 0.04019503518939018, + "rewards/format_reward/mean": 0.10499999970197678, + "rewards/format_reward/std": 0.13398344144225122, + "rewards/security_audit_reward/mean": -0.5, + "rewards/security_audit_reward/std": 0.0, + "step": 30, + "step_time": 38.56538706479987 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.4, + "completions/max_length": 504.2, + "completions/max_terminated_length": 454.6, + "completions/mean_length": 421.25, + "completions/mean_terminated_length": 386.0, + "completions/min_length": 328.6, + "completions/min_terminated_length": 328.6, + "entropy": 1.3522289156913758, + "epoch": 0.2147239263803681, + "frac_reward_zero_std": 0.3, + "grad_norm": 3.4385552406311035, + "learning_rate": 4.4785276073619634e-07, + "loss": -0.06348788738250732, + "num_tokens": 126953.0, + "reward": -0.32824997901916503, + "reward_std": 0.03220053892582655, + "rewards/format_reward/mean": 0.07250000201165677, + "rewards/format_reward/std": 0.10733511671423912, + "rewards/security_audit_reward/mean": -0.5, + "rewards/security_audit_reward/std": 0.0, + "step": 35, + "step_time": 37.87626404739986 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.55, + "completions/max_length": 505.2, + "completions/max_terminated_length": 334.4, + "completions/mean_length": 414.9, + "completions/mean_terminated_length": 240.3, + "completions/min_length": 243.0, + "completions/min_terminated_length": 140.6, + "entropy": 1.230024951696396, + "epoch": 0.24539877300613497, + "frac_reward_zero_std": 0.3, + "grad_norm": 4.400479793548584, + "learning_rate": 4.401840490797546e-07, + "loss": 0.11927952766418456, + "num_tokens": 144785.0, + "reward": -0.2897499829530716, + "reward_std": 0.12973095811903476, + "rewards/format_reward/mean": 0.14250000044703484, + "rewards/format_reward/std": 0.14365934804081917, + "rewards/security_audit_reward/mean": -0.475, + "rewards/security_audit_reward/std": 0.13164966106414794, + "step": 40, + "step_time": 37.8069536416001 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.6, + "completions/max_length": 472.2, + "completions/max_terminated_length": 190.6, + "completions/mean_length": 412.7, + "completions/mean_terminated_length": 160.85, + "completions/min_length": 333.8, + "completions/min_terminated_length": 129.0, + "entropy": 1.2133947968482972, + "epoch": 0.27607361963190186, + "frac_reward_zero_std": 0.1, + "grad_norm": 4.325937271118164, + "learning_rate": 4.3251533742331285e-07, + "loss": 0.025146520137786864, + "num_tokens": 162443.0, + "reward": -0.1574999876320362, + "reward_std": 0.2636621415615082, + "rewards/format_reward/mean": 0.24500001072883607, + "rewards/format_reward/std": 0.23762110471725464, + "rewards/security_audit_reward/mean": -0.32999999523162843, + "rewards/security_audit_reward/std": 0.28574271202087403, + "step": 45, + "step_time": 34.90182834920015 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.55, + "completions/max_length": 512.0, + "completions/max_terminated_length": 308.6, + "completions/mean_length": 397.05, + "completions/mean_terminated_length": 259.2, + "completions/min_length": 203.0, + "completions/min_terminated_length": 203.0, + "entropy": 1.4294291973114013, + "epoch": 0.3067484662576687, + "frac_reward_zero_std": 0.4, + "grad_norm": 3.9505743980407715, + "learning_rate": 4.2484662576687116e-07, + "loss": -0.08058007955551147, + "num_tokens": 180200.0, + "reward": -0.29249998927116394, + "reward_std": 0.10127481501549482, + "rewards/format_reward/mean": 0.0750000026077032, + "rewards/format_reward/std": 0.1127780631184578, + "rewards/security_audit_reward/mean": -0.45, + "rewards/security_audit_reward/std": 0.1, + "step": 50, + "step_time": 38.750808009400046 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.5, + "completions/max_length": 512.0, + "completions/max_terminated_length": 389.8, + "completions/mean_length": 404.65, + "completions/mean_terminated_length": 297.43333740234374, + "completions/min_length": 192.8, + "completions/min_terminated_length": 192.8, + "entropy": 1.2564165532588958, + "epoch": 0.3374233128834356, + "frac_reward_zero_std": 0.3, + "grad_norm": 3.3762269020080566, + "learning_rate": 4.171779141104294e-07, + "loss": -0.030467823147773743, + "num_tokens": 198109.0, + "reward": -0.2542499825358391, + "reward_std": 0.07489922866225243, + "rewards/format_reward/mean": 0.20250000841915608, + "rewards/format_reward/std": 0.1368803471326828, + "rewards/security_audit_reward/mean": -0.45, + "rewards/security_audit_reward/std": 0.05773502588272095, + "step": 55, + "step_time": 38.411275500399825 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.35, + "completions/max_length": 496.2, + "completions/max_terminated_length": 448.8, + "completions/mean_length": 394.3, + "completions/mean_terminated_length": 358.6166687011719, + "completions/min_length": 285.4, + "completions/min_terminated_length": 285.4, + "entropy": 1.2620218694210052, + "epoch": 0.36809815950920244, + "frac_reward_zero_std": 0.2, + "grad_norm": 2.8227944374084473, + "learning_rate": 4.095092024539877e-07, + "loss": 0.039707571268081665, + "num_tokens": 215747.0, + "reward": -0.2729999750852585, + "reward_std": 0.13599938787519933, + "rewards/format_reward/mean": 0.14000000432133675, + "rewards/format_reward/std": 0.14343783408403396, + "rewards/security_audit_reward/mean": -0.45, + "rewards/security_audit_reward/std": 0.1393846869468689, + "step": 60, + "step_time": 37.50645367139987 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.35, + "completions/max_length": 484.8, + "completions/max_terminated_length": 334.8, + "completions/mean_length": 381.35, + "completions/mean_terminated_length": 255.98333740234375, + "completions/min_length": 275.6, + "completions/min_terminated_length": 173.2, + "entropy": 1.2798833012580872, + "epoch": 0.3987730061349693, + "frac_reward_zero_std": 0.2, + "grad_norm": 3.4819753170013428, + "learning_rate": 4.01840490797546e-07, + "loss": -0.06275686025619506, + "num_tokens": 233162.0, + "reward": -0.201749986410141, + "reward_std": 0.2016347900032997, + "rewards/format_reward/mean": 0.20250000804662704, + "rewards/format_reward/std": 0.22137173414230346, + "rewards/security_audit_reward/mean": -0.375, + "rewards/security_audit_reward/std": 0.20773502588272094, + "step": 65, + "step_time": 37.139902984000216 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.6, + "completions/max_length": 512.0, + "completions/max_terminated_length": 289.8, + "completions/mean_length": 408.45, + "completions/mean_terminated_length": 251.7, + "completions/min_length": 207.0, + "completions/min_terminated_length": 207.0, + "entropy": 1.2134525895118713, + "epoch": 0.4294478527607362, + "frac_reward_zero_std": 0.3, + "grad_norm": 4.019806861877441, + "learning_rate": 3.941717791411043e-07, + "loss": 0.08099154829978943, + "num_tokens": 251321.0, + "reward": -0.27599998414516447, + "reward_std": 0.08945702444761991, + "rewards/format_reward/mean": 0.1300000037997961, + "rewards/format_reward/std": 0.1645726040005684, + "rewards/security_audit_reward/mean": -0.45, + "rewards/security_audit_reward/std": 0.05773502588272095, + "step": 70, + "step_time": 38.02127088899997 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.5, + "completions/max_length": 459.8, + "completions/max_terminated_length": 206.2, + "completions/mean_length": 348.9, + "completions/mean_terminated_length": 176.9, + "completions/min_length": 148.6, + "completions/min_terminated_length": 148.6, + "entropy": 1.3179432690143584, + "epoch": 0.4601226993865031, + "frac_reward_zero_std": 0.0, + "grad_norm": 6.28598690032959, + "learning_rate": 3.8650306748466255e-07, + "loss": -0.11171818971633911, + "num_tokens": 267725.0, + "reward": -0.19474998638033866, + "reward_std": 0.17031802013516426, + "rewards/format_reward/mean": 0.23750000447034836, + "rewards/format_reward/std": 0.17114628925919534, + "rewards/security_audit_reward/mean": -0.3800000011920929, + "rewards/security_audit_reward/std": 0.180902099609375, + "step": 75, + "step_time": 34.450158203000136 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.5, + "completions/max_length": 512.0, + "completions/max_terminated_length": 332.8, + "completions/mean_length": 400.75, + "completions/mean_terminated_length": 289.06666870117186, + "completions/min_length": 251.0, + "completions/min_terminated_length": 251.0, + "entropy": 1.1514661133289337, + "epoch": 0.49079754601226994, + "frac_reward_zero_std": 0.4, + "grad_norm": 2.8479247093200684, + "learning_rate": 3.788343558282208e-07, + "loss": 0.03145935535430908, + "num_tokens": 285726.0, + "reward": -0.2569999933242798, + "reward_std": 0.15333212018013, + "rewards/format_reward/mean": 0.1350000023841858, + "rewards/format_reward/std": 0.18636635541915894, + "rewards/security_audit_reward/mean": -0.425, + "rewards/security_audit_reward/std": 0.15, + "step": 80, + "step_time": 38.869779922999626 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.5, + "completions/max_length": 512.0, + "completions/max_terminated_length": 321.2, + "completions/mean_length": 398.5, + "completions/mean_terminated_length": 253.0, + "completions/min_length": 183.0, + "completions/min_terminated_length": 183.0, + "entropy": 1.244500571489334, + "epoch": 0.5214723926380368, + "frac_reward_zero_std": 0.3, + "grad_norm": 2.146970272064209, + "learning_rate": 3.7116564417177916e-07, + "loss": 0.06171210408210755, + "num_tokens": 304148.0, + "reward": -0.22524999380111693, + "reward_std": 0.19191497713327407, + "rewards/format_reward/mean": 0.18250000327825547, + "rewards/format_reward/std": 0.19969657957553863, + "rewards/security_audit_reward/mean": -0.4, + "rewards/security_audit_reward/std": 0.2, + "step": 85, + "step_time": 39.297288996000134 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.55, + "completions/max_length": 512.0, + "completions/max_terminated_length": 360.0, + "completions/mean_length": 419.5, + "completions/mean_terminated_length": 325.6, + "completions/min_length": 291.2, + "completions/min_terminated_length": 291.2, + "entropy": 1.206581747531891, + "epoch": 0.5521472392638037, + "frac_reward_zero_std": 0.2, + "grad_norm": 2.1158456802368164, + "learning_rate": 3.634969325153374e-07, + "loss": -0.06664568185806274, + "num_tokens": 321680.0, + "reward": -0.23324998915195466, + "reward_std": 0.1919491995126009, + "rewards/format_reward/mean": 0.1675000049173832, + "rewards/format_reward/std": 0.19863576367497443, + "rewards/security_audit_reward/mean": -0.40499999523162844, + "rewards/security_audit_reward/std": 0.1899999976158142, + "step": 90, + "step_time": 38.49561594039933 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.4, + "completions/max_length": 469.2, + "completions/max_terminated_length": 364.0, + "completions/mean_length": 383.2, + "completions/mean_terminated_length": 291.8333374023438, + "completions/min_length": 217.6, + "completions/min_terminated_length": 217.6, + "entropy": 1.217250692844391, + "epoch": 0.5828220858895705, + "frac_reward_zero_std": 0.4, + "grad_norm": 4.098232269287109, + "learning_rate": 3.558282208588957e-07, + "loss": 0.05211906433105469, + "num_tokens": 339350.0, + "reward": -0.2119999945163727, + "reward_std": 0.19894140996038914, + "rewards/format_reward/mean": 0.1799999989569187, + "rewards/format_reward/std": 0.23350853994488716, + "rewards/security_audit_reward/mean": -0.37999999821186065, + "rewards/security_audit_reward/std": 0.1911805212497711, + "step": 95, + "step_time": 35.83687614579994 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.5, + "completions/max_length": 512.0, + "completions/max_terminated_length": 345.4, + "completions/mean_length": 377.45, + "completions/mean_terminated_length": 290.6333343505859, + "completions/min_length": 240.4, + "completions/min_terminated_length": 240.4, + "entropy": 1.2783292949199676, + "epoch": 0.6134969325153374, + "frac_reward_zero_std": 0.3, + "grad_norm": 2.361516237258911, + "learning_rate": 3.48159509202454e-07, + "loss": 0.06258203387260437, + "num_tokens": 356239.0, + "reward": -0.20649999231100083, + "reward_std": 0.18195689767599105, + "rewards/format_reward/mean": 0.24499999433755876, + "rewards/format_reward/std": 0.19310407042503358, + "rewards/security_audit_reward/mean": -0.4, + "rewards/security_audit_reward/std": 0.2, + "step": 100, + "step_time": 38.47846096040011 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.3, + "completions/max_length": 463.2, + "completions/max_terminated_length": 394.6, + "completions/mean_length": 335.8, + "completions/mean_terminated_length": 268.3, + "completions/min_length": 139.6, + "completions/min_terminated_length": 139.6, + "entropy": 1.2529696226119995, + "epoch": 0.6441717791411042, + "frac_reward_zero_std": 0.2, + "grad_norm": 3.356074094772339, + "learning_rate": 3.4049079754601224e-07, + "loss": 0.003340443968772888, + "num_tokens": 373237.0, + "reward": -0.2567499876022339, + "reward_std": 0.27417250275611876, + "rewards/format_reward/mean": 0.14750000461935997, + "rewards/format_reward/std": 0.19759280756115913, + "rewards/security_audit_reward/mean": -0.4299999952316284, + "rewards/security_audit_reward/std": 0.3186576545238495, + "step": 105, + "step_time": 35.43083410320014 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.45, + "completions/max_length": 512.0, + "completions/max_terminated_length": 372.8, + "completions/mean_length": 419.15, + "completions/mean_terminated_length": 319.2666748046875, + "completions/min_length": 266.8, + "completions/min_terminated_length": 266.8, + "entropy": 1.1685741186141967, + "epoch": 0.6748466257668712, + "frac_reward_zero_std": 0.2, + "grad_norm": 4.318619728088379, + "learning_rate": 3.3282208588957055e-07, + "loss": -0.026089027523994446, + "num_tokens": 391784.0, + "reward": -0.2662499874830246, + "reward_std": 0.07884115856140853, + "rewards/format_reward/mean": 0.16250000558793545, + "rewards/format_reward/std": 0.14070439487695693, + "rewards/security_audit_reward/mean": -0.45, + "rewards/security_audit_reward/std": 0.05773502588272095, + "step": 110, + "step_time": 38.89281254739999 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.3, + "completions/max_length": 468.0, + "completions/max_terminated_length": 337.6, + "completions/mean_length": 330.1, + "completions/mean_terminated_length": 240.4166687011719, + "completions/min_length": 160.8, + "completions/min_terminated_length": 160.8, + "entropy": 1.2954379856586455, + "epoch": 0.7055214723926381, + "frac_reward_zero_std": 0.2, + "grad_norm": 3.293928384780884, + "learning_rate": 3.251533742331288e-07, + "loss": 0.17276796102523803, + "num_tokens": 408446.0, + "reward": -0.22849999666213988, + "reward_std": 0.1390242099761963, + "rewards/format_reward/mean": 0.2300000011920929, + "rewards/format_reward/std": 0.23302415013313293, + "rewards/security_audit_reward/mean": -0.425, + "rewards/security_audit_reward/std": 0.10773502588272095, + "step": 115, + "step_time": 35.73082293679981 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.55, + "completions/max_length": 512.0, + "completions/max_terminated_length": 281.2, + "completions/mean_length": 400.5, + "completions/mean_terminated_length": 248.73333740234375, + "completions/min_length": 211.8, + "completions/min_terminated_length": 211.8, + "entropy": 1.2283548831939697, + "epoch": 0.7361963190184049, + "frac_reward_zero_std": 0.2, + "grad_norm": 2.632479190826416, + "learning_rate": 3.174846625766871e-07, + "loss": 0.05111231803894043, + "num_tokens": 426822.0, + "reward": -0.22074998915195465, + "reward_std": 0.15957241374999284, + "rewards/format_reward/mean": 0.1975000012665987, + "rewards/format_reward/std": 0.1847505249083042, + "rewards/security_audit_reward/mean": -0.4, + "rewards/security_audit_reward/std": 0.15773502588272095, + "step": 120, + "step_time": 39.14503100519996 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.5, + "completions/max_length": 512.0, + "completions/max_terminated_length": 340.2, + "completions/mean_length": 401.4, + "completions/mean_terminated_length": 236.03333740234376, + "completions/min_length": 245.2, + "completions/min_terminated_length": 142.8, + "entropy": 1.307636547088623, + "epoch": 0.7668711656441718, + "frac_reward_zero_std": 0.3, + "grad_norm": 5.566491603851318, + "learning_rate": 3.0981595092024537e-07, + "loss": 0.003215853124856949, + "num_tokens": 444322.0, + "reward": -0.11199999079108239, + "reward_std": 0.2506739288568497, + "rewards/format_reward/mean": 0.2449999988079071, + "rewards/format_reward/std": 0.20622505843639374, + "rewards/security_audit_reward/mean": -0.26500000059604645, + "rewards/security_audit_reward/std": 0.278915548324585, + "step": 125, + "step_time": 38.70229864360026 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.35, + "completions/max_length": 512.0, + "completions/max_terminated_length": 390.6, + "completions/mean_length": 367.0, + "completions/mean_terminated_length": 264.0000061035156, + "completions/min_length": 123.2, + "completions/min_terminated_length": 123.2, + "entropy": 1.248900693655014, + "epoch": 0.7975460122699386, + "frac_reward_zero_std": 0.2, + "grad_norm": 4.395384311676025, + "learning_rate": 3.021472392638036e-07, + "loss": 0.06482647061347961, + "num_tokens": 461894.0, + "reward": -0.2042499899864197, + "reward_std": 0.17003463432192803, + "rewards/format_reward/mean": 0.2525000125169754, + "rewards/format_reward/std": 0.21560870110988617, + "rewards/security_audit_reward/mean": -0.4, + "rewards/security_audit_reward/std": 0.15773502588272095, + "step": 130, + "step_time": 39.174271353800215 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.35, + "completions/max_length": 512.0, + "completions/max_terminated_length": 370.0, + "completions/mean_length": 359.95, + "completions/mean_terminated_length": 276.9666687011719, + "completions/min_length": 203.8, + "completions/min_terminated_length": 203.8, + "entropy": 1.3299469709396363, + "epoch": 0.8282208588957055, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.3037519454956055, + "learning_rate": 2.94478527607362e-07, + "loss": 0.026153716444969177, + "num_tokens": 478783.0, + "reward": -0.19949999153614045, + "reward_std": 0.15296672135591508, + "rewards/format_reward/mean": 0.24500000178813935, + "rewards/format_reward/std": 0.2273508906364441, + "rewards/security_audit_reward/mean": -0.39000000059604645, + "rewards/security_audit_reward/std": 0.12891554832458496, + "step": 135, + "step_time": 38.658669441000164 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.45, + "completions/max_length": 512.0, + "completions/max_terminated_length": 354.0, + "completions/mean_length": 380.3, + "completions/mean_terminated_length": 271.6666687011719, + "completions/min_length": 206.4, + "completions/min_terminated_length": 206.4, + "entropy": 1.0997539341449738, + "epoch": 0.8588957055214724, + "frac_reward_zero_std": 0.2, + "grad_norm": 2.3693976402282715, + "learning_rate": 2.8680981595092024e-07, + "loss": -0.01876506209373474, + "num_tokens": 496243.0, + "reward": -0.17974998727440833, + "reward_std": 0.18710523881018162, + "rewards/format_reward/mean": 0.21750000193715097, + "rewards/format_reward/std": 0.1843859799206257, + "rewards/security_audit_reward/mean": -0.35, + "rewards/security_audit_reward/std": 0.20347774028778076, + "step": 140, + "step_time": 39.171343391999834 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.35, + "completions/max_length": 456.2, + "completions/max_terminated_length": 355.2, + "completions/mean_length": 348.35, + "completions/mean_terminated_length": 270.3, + "completions/min_length": 164.8, + "completions/min_terminated_length": 164.8, + "entropy": 1.2017314374446868, + "epoch": 0.8895705521472392, + "frac_reward_zero_std": 0.3, + "grad_norm": 4.230531215667725, + "learning_rate": 2.791411042944785e-07, + "loss": 0.0029310762882232668, + "num_tokens": 513422.0, + "reward": -0.1637499898672104, + "reward_std": 0.2463478922843933, + "rewards/format_reward/mean": 0.2475000023841858, + "rewards/format_reward/std": 0.2085829883813858, + "rewards/security_audit_reward/mean": -0.34000000059604646, + "rewards/security_audit_reward/std": 0.26830023527145386, + "step": 145, + "step_time": 34.95672115479992 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.25, + "completions/max_length": 482.4, + "completions/max_terminated_length": 431.6, + "completions/mean_length": 355.35, + "completions/mean_terminated_length": 319.4166687011719, + "completions/min_length": 212.0, + "completions/min_terminated_length": 212.0, + "entropy": 1.258862280845642, + "epoch": 0.9202453987730062, + "frac_reward_zero_std": 0.2, + "grad_norm": 6.071740627288818, + "learning_rate": 2.714723926380368e-07, + "loss": 0.0822126567363739, + "num_tokens": 530643.0, + "reward": -0.1912499874830246, + "reward_std": 0.1670845106244087, + "rewards/format_reward/mean": 0.27249999940395353, + "rewards/format_reward/std": 0.1733592666685581, + "rewards/security_audit_reward/mean": -0.39000000059604645, + "rewards/security_audit_reward/std": 0.17118052244186402, + "step": 150, + "step_time": 37.038782767599876 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.2, + "completions/max_length": 442.2, + "completions/max_terminated_length": 316.0, + "completions/mean_length": 283.75, + "completions/mean_terminated_length": 237.85000305175782, + "completions/min_length": 166.2, + "completions/min_terminated_length": 166.2, + "entropy": 1.4853489220142364, + "epoch": 0.950920245398773, + "frac_reward_zero_std": 0.0, + "grad_norm": 5.2570695877075195, + "learning_rate": 2.6380368098159506e-07, + "loss": 0.11004064083099366, + "num_tokens": 545966.0, + "reward": -0.15949999541044235, + "reward_std": 0.19192611873149873, + "rewards/format_reward/mean": 0.28500000238418577, + "rewards/format_reward/std": 0.22434256076812745, + "rewards/security_audit_reward/mean": -0.35, + "rewards/security_audit_reward/std": 0.2, + "step": 155, + "step_time": 33.86747411140077 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.55, + "completions/max_length": 512.0, + "completions/max_terminated_length": 340.8, + "completions/mean_length": 411.45, + "completions/mean_terminated_length": 265.9666687011719, + "completions/min_length": 192.0, + "completions/min_terminated_length": 192.0, + "entropy": 1.0828768193721772, + "epoch": 0.9815950920245399, + "frac_reward_zero_std": 0.2, + "grad_norm": 2.6537587642669678, + "learning_rate": 2.5613496932515337e-07, + "loss": 0.03556116819381714, + "num_tokens": 563683.0, + "reward": -0.20099999010562897, + "reward_std": 0.1888158166781068, + "rewards/format_reward/mean": 0.24000000059604645, + "rewards/format_reward/std": 0.16870398968458175, + "rewards/security_audit_reward/mean": -0.3899999976158142, + "rewards/security_audit_reward/std": 0.2199999988079071, + "step": 160, + "step_time": 38.665304075799575 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.3, + "completions/max_length": 483.2, + "completions/max_terminated_length": 393.2, + "completions/mean_length": 359.2, + "completions/mean_terminated_length": 303.3666687011719, + "completions/min_length": 178.2, + "completions/min_terminated_length": 178.2, + "entropy": 1.1811485469341279, + "epoch": 1.0122699386503067, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.203860282897949, + "learning_rate": 2.4846625766871163e-07, + "loss": -0.02532302737236023, + "num_tokens": 580183.0, + "reward": -0.1227499857544899, + "reward_std": 0.2651766210794449, + "rewards/format_reward/mean": 0.2675000011920929, + "rewards/format_reward/std": 0.24115291833877564, + "rewards/security_audit_reward/mean": -0.2899999976158142, + "rewards/security_audit_reward/std": 0.2812127649784088, + "step": 165, + "step_time": 36.34888075860035 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.4, + "completions/max_length": 512.0, + "completions/max_terminated_length": 345.8, + "completions/mean_length": 358.5, + "completions/mean_terminated_length": 235.96666870117187, + "completions/min_length": 123.0, + "completions/min_terminated_length": 123.0, + "entropy": 1.2863860994577407, + "epoch": 1.0429447852760736, + "frac_reward_zero_std": 0.2, + "grad_norm": 2.71185302734375, + "learning_rate": 2.4079754601226994e-07, + "loss": 0.12254136800765991, + "num_tokens": 597345.0, + "reward": -0.20274999886751174, + "reward_std": 0.1825057201087475, + "rewards/format_reward/mean": 0.25750000327825545, + "rewards/format_reward/std": 0.19183385372161865, + "rewards/security_audit_reward/mean": -0.4, + "rewards/security_audit_reward/std": 0.19711971282958984, + "step": 170, + "step_time": 38.96664929399922 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.5, + "completions/max_length": 512.0, + "completions/max_terminated_length": 323.4, + "completions/mean_length": 407.1, + "completions/mean_terminated_length": 234.40000610351564, + "completions/min_length": 242.8, + "completions/min_terminated_length": 140.4, + "entropy": 1.179810070991516, + "epoch": 1.0736196319018405, + "frac_reward_zero_std": 0.2, + "grad_norm": 4.413055419921875, + "learning_rate": 2.331288343558282e-07, + "loss": 0.041037318110466, + "num_tokens": 615063.0, + "reward": -0.20374999046325684, + "reward_std": 0.2052689865231514, + "rewards/format_reward/mean": 0.31250000894069674, + "rewards/format_reward/std": 0.22553626000881194, + "rewards/security_audit_reward/mean": -0.425, + "rewards/security_audit_reward/std": 0.23164966106414794, + "step": 175, + "step_time": 39.00492364500023 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.3, + "completions/max_length": 511.6, + "completions/max_terminated_length": 424.0, + "completions/mean_length": 399.45, + "completions/mean_terminated_length": 347.9166748046875, + "completions/min_length": 262.6, + "completions/min_terminated_length": 262.6, + "entropy": 1.1026120364665986, + "epoch": 1.1042944785276074, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.408414840698242, + "learning_rate": 2.254601226993865e-07, + "loss": 0.058766734600067136, + "num_tokens": 632696.0, + "reward": -0.16599998623132706, + "reward_std": 0.26377752125263215, + "rewards/format_reward/mean": 0.24000000655651094, + "rewards/format_reward/std": 0.21778101623058319, + "rewards/security_audit_reward/mean": -0.34000000059604646, + "rewards/security_audit_reward/std": 0.3105652093887329, + "step": 180, + "step_time": 39.09616019519963 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.2, + "completions/max_length": 477.4, + "completions/max_terminated_length": 421.6, + "completions/mean_length": 322.45, + "completions/mean_terminated_length": 273.1000030517578, + "completions/min_length": 156.0, + "completions/min_terminated_length": 156.0, + "entropy": 1.2888785600662231, + "epoch": 1.1349693251533743, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.6877431869506836, + "learning_rate": 2.1779141104294476e-07, + "loss": -0.0771723210811615, + "num_tokens": 649353.0, + "reward": -0.1799999952316284, + "reward_std": 0.3134476348757744, + "rewards/format_reward/mean": 0.2750000089406967, + "rewards/format_reward/std": 0.24135999679565429, + "rewards/security_audit_reward/mean": -0.375, + "rewards/security_audit_reward/std": 0.3593961834907532, + "step": 185, + "step_time": 36.71157897000012 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.3, + "completions/max_length": 495.0, + "completions/max_terminated_length": 351.0, + "completions/mean_length": 328.8, + "completions/mean_terminated_length": 243.7, + "completions/min_length": 164.8, + "completions/min_terminated_length": 164.8, + "entropy": 1.4585140287876128, + "epoch": 1.165644171779141, + "frac_reward_zero_std": 0.2, + "grad_norm": 4.987306118011475, + "learning_rate": 2.1012269938650307e-07, + "loss": -0.15080010890960693, + "num_tokens": 665513.0, + "reward": -0.050499990582466125, + "reward_std": 0.2684710592031479, + "rewards/format_reward/mean": 0.31000000387430193, + "rewards/format_reward/std": 0.1994625985622406, + "rewards/security_audit_reward/mean": -0.20500000119209288, + "rewards/security_audit_reward/std": 0.31255176067352297, + "step": 190, + "step_time": 37.63543628939988 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.5, + "completions/max_length": 512.0, + "completions/max_terminated_length": 307.8, + "completions/mean_length": 391.75, + "completions/mean_terminated_length": 253.06666870117186, + "completions/min_length": 178.0, + "completions/min_terminated_length": 178.0, + "entropy": 1.1293343544006347, + "epoch": 1.196319018404908, + "frac_reward_zero_std": 0.1, + "grad_norm": 5.247244358062744, + "learning_rate": 2.0245398773006135e-07, + "loss": -0.04229157567024231, + "num_tokens": 683268.0, + "reward": -0.10224998965859414, + "reward_std": 0.19266743455082178, + "rewards/format_reward/mean": 0.3124999929219484, + "rewards/format_reward/std": 0.1390557773411274, + "rewards/security_audit_reward/mean": -0.2800000011920929, + "rewards/security_audit_reward/std": 0.23863712549209595, + "step": 195, + "step_time": 38.93936442300037 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.4, + "completions/max_length": 485.0, + "completions/max_terminated_length": 346.2, + "completions/mean_length": 364.6, + "completions/mean_terminated_length": 265.8166748046875, + "completions/min_length": 165.8, + "completions/min_terminated_length": 165.8, + "entropy": 0.8287177711725235, + "epoch": 1.2269938650306749, + "frac_reward_zero_std": 0.1, + "grad_norm": 2.4230945110321045, + "learning_rate": 1.9478527607361963e-07, + "loss": -0.05633368492126465, + "num_tokens": 700760.0, + "reward": -0.1807499848306179, + "reward_std": 0.18529897555708885, + "rewards/format_reward/mean": 0.3075000137090683, + "rewards/format_reward/std": 0.15467575192451477, + "rewards/security_audit_reward/mean": -0.39000000059604645, + "rewards/security_audit_reward/std": 0.2105652093887329, + "step": 200, + "step_time": 37.065423558799736 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.25, + "completions/max_length": 431.6, + "completions/max_terminated_length": 338.8, + "completions/mean_length": 284.1, + "completions/mean_terminated_length": 225.65, + "completions/min_length": 119.4, + "completions/min_terminated_length": 119.4, + "entropy": 1.2736368715763091, + "epoch": 1.2576687116564418, + "frac_reward_zero_std": 0.1, + "grad_norm": 4.797567367553711, + "learning_rate": 1.8711656441717791e-07, + "loss": 0.08297693133354186, + "num_tokens": 716344.0, + "reward": -0.07274999544024467, + "reward_std": 0.24350565671920776, + "rewards/format_reward/mean": 0.31749999821186065, + "rewards/format_reward/std": 0.19669782146811485, + "rewards/security_audit_reward/mean": -0.23999999985098838, + "rewards/security_audit_reward/std": 0.2692204549908638, + "step": 205, + "step_time": 33.11877055760014 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.25, + "completions/max_length": 484.0, + "completions/max_terminated_length": 403.2, + "completions/mean_length": 335.45, + "completions/mean_terminated_length": 276.4500030517578, + "completions/min_length": 173.8, + "completions/min_terminated_length": 173.8, + "entropy": 1.1084223449230195, + "epoch": 1.2883435582822087, + "frac_reward_zero_std": 0.1, + "grad_norm": 2.4603023529052734, + "learning_rate": 1.7944785276073617e-07, + "loss": 0.07945090532302856, + "num_tokens": 733245.0, + "reward": -0.13774999380111694, + "reward_std": 0.2730386942625046, + "rewards/format_reward/mean": 0.2174999989569187, + "rewards/format_reward/std": 0.22229814901947975, + "rewards/security_audit_reward/mean": -0.29000000059604647, + "rewards/security_audit_reward/std": 0.3105652093887329, + "step": 210, + "step_time": 37.03591289120122 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.4, + "completions/max_length": 512.0, + "completions/max_terminated_length": 361.8, + "completions/mean_length": 345.0, + "completions/mean_terminated_length": 261.3000030517578, + "completions/min_length": 184.8, + "completions/min_terminated_length": 184.8, + "entropy": 1.2274070978164673, + "epoch": 1.3190184049079754, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.573819637298584, + "learning_rate": 1.7177914110429448e-07, + "loss": -0.07497722506523133, + "num_tokens": 749917.0, + "reward": -0.01174999624490738, + "reward_std": 0.3002330154180527, + "rewards/format_reward/mean": 0.3225000023841858, + "rewards/format_reward/std": 0.1751384623348713, + "rewards/security_audit_reward/mean": -0.15499999821186067, + "rewards/security_audit_reward/std": 0.3648489773273468, + "step": 215, + "step_time": 38.89012140319937 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.2, + "completions/max_length": 481.8, + "completions/max_terminated_length": 344.6, + "completions/mean_length": 292.3, + "completions/mean_terminated_length": 234.26666870117188, + "completions/min_length": 122.2, + "completions/min_terminated_length": 122.2, + "entropy": 1.1711494624614716, + "epoch": 1.3496932515337423, + "frac_reward_zero_std": 0.1, + "grad_norm": 3.879939556121826, + "learning_rate": 1.6411042944785276e-07, + "loss": 0.06901218891143798, + "num_tokens": 765457.0, + "reward": -0.2002499908208847, + "reward_std": 0.19745510853827, + "rewards/format_reward/mean": 0.20749999657273294, + "rewards/format_reward/std": 0.2098293460905552, + "rewards/security_audit_reward/mean": -0.375, + "rewards/security_audit_reward/std": 0.20773502588272094, + "step": 220, + "step_time": 36.50884771559977 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.35, + "completions/max_length": 451.4, + "completions/max_terminated_length": 281.8, + "completions/mean_length": 310.85, + "completions/mean_terminated_length": 210.58333435058594, + "completions/min_length": 144.0, + "completions/min_terminated_length": 144.0, + "entropy": 1.4326449751853942, + "epoch": 1.3803680981595092, + "frac_reward_zero_std": 0.0, + "grad_norm": 5.483170986175537, + "learning_rate": 1.5644171779141104e-07, + "loss": -0.03490494191646576, + "num_tokens": 782226.0, + "reward": -0.14649999141693115, + "reward_std": 0.19791007936000823, + "rewards/format_reward/mean": 0.3049999952316284, + "rewards/format_reward/std": 0.19433450996875762, + "rewards/security_audit_reward/mean": -0.3399999998509884, + "rewards/security_audit_reward/std": 0.22000000029802322, + "step": 225, + "step_time": 35.079213985799655 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.35, + "completions/max_length": 511.2, + "completions/max_terminated_length": 315.2, + "completions/mean_length": 338.15, + "completions/mean_terminated_length": 226.28333435058593, + "completions/min_length": 134.8, + "completions/min_terminated_length": 134.8, + "entropy": 1.1364098012447357, + "epoch": 1.4110429447852761, + "frac_reward_zero_std": 0.2, + "grad_norm": 3.3578364849090576, + "learning_rate": 1.4877300613496933e-07, + "loss": 0.0896155834197998, + "num_tokens": 798571.0, + "reward": -0.11574998870491982, + "reward_std": 0.19651760943233967, + "rewards/format_reward/mean": 0.2674999989569187, + "rewards/format_reward/std": 0.15046989992260934, + "rewards/security_audit_reward/mean": -0.27999999821186067, + "rewards/security_audit_reward/std": 0.2297215759754181, + "step": 230, + "step_time": 38.61798697480081 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.35, + "completions/max_length": 505.4, + "completions/max_terminated_length": 385.4, + "completions/mean_length": 360.65, + "completions/mean_terminated_length": 295.23333740234375, + "completions/min_length": 210.2, + "completions/min_terminated_length": 210.2, + "entropy": 1.0565216183662414, + "epoch": 1.441717791411043, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.266097068786621, + "learning_rate": 1.4110429447852758e-07, + "loss": 0.07759050726890564, + "num_tokens": 815570.0, + "reward": -0.06749999299645423, + "reward_std": 0.27374918162822726, + "rewards/format_reward/mean": 0.37000001072883604, + "rewards/format_reward/std": 0.1865294199436903, + "rewards/security_audit_reward/mean": -0.2550000011920929, + "rewards/security_audit_reward/std": 0.32802181243896483, + "step": 235, + "step_time": 38.44030983600023 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.3, + "completions/max_length": 495.6, + "completions/max_terminated_length": 352.6, + "completions/mean_length": 336.05, + "completions/mean_terminated_length": 243.60000915527343, + "completions/min_length": 145.8, + "completions/min_terminated_length": 145.8, + "entropy": 1.417020809650421, + "epoch": 1.4723926380368098, + "frac_reward_zero_std": 0.2, + "grad_norm": 4.767548084259033, + "learning_rate": 1.334355828220859e-07, + "loss": 0.03671485185623169, + "num_tokens": 831713.0, + "reward": -0.12849999219179153, + "reward_std": 0.19381159394979477, + "rewards/format_reward/mean": 0.3300000041723251, + "rewards/format_reward/std": 0.20168980173766612, + "rewards/security_audit_reward/mean": -0.325, + "rewards/security_audit_reward/std": 0.20773502588272094, + "step": 240, + "step_time": 37.49174958900003 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.6, + "completions/max_length": 512.0, + "completions/max_terminated_length": 343.0, + "completions/mean_length": 417.3, + "completions/mean_terminated_length": 293.7, + "completions/min_length": 246.8, + "completions/min_terminated_length": 246.8, + "entropy": 1.0088598132133484, + "epoch": 1.5030674846625767, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.1936607360839844, + "learning_rate": 1.2576687116564417e-07, + "loss": -0.03240810632705689, + "num_tokens": 849855.0, + "reward": -0.11324999332427979, + "reward_std": 0.2602782666683197, + "rewards/format_reward/mean": 0.3224999994039536, + "rewards/format_reward/std": 0.20757876634597777, + "rewards/security_audit_reward/mean": -0.3, + "rewards/security_audit_reward/std": 0.3154700517654419, + "step": 245, + "step_time": 39.07302968719996 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.2, + "completions/max_length": 459.6, + "completions/max_terminated_length": 428.2, + "completions/mean_length": 325.35, + "completions/mean_terminated_length": 277.5833374023438, + "completions/min_length": 137.0, + "completions/min_terminated_length": 137.0, + "entropy": 1.1867628961801528, + "epoch": 1.5337423312883436, + "frac_reward_zero_std": 0.1, + "grad_norm": 3.9449942111968994, + "learning_rate": 1.1809815950920244e-07, + "loss": -0.005416367202997208, + "num_tokens": 866330.0, + "reward": -0.08199999034404755, + "reward_std": 0.2747137784957886, + "rewards/format_reward/mean": 0.3099999874830246, + "rewards/format_reward/std": 0.17935641929507257, + "rewards/security_audit_reward/mean": -0.25, + "rewards/security_audit_reward/std": 0.33094010353088377, + "step": 250, + "step_time": 35.27377968320034 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.25, + "completions/max_length": 450.0, + "completions/max_terminated_length": 311.2, + "completions/mean_length": 283.8, + "completions/mean_terminated_length": 202.06666870117186, + "completions/min_length": 128.0, + "completions/min_terminated_length": 128.0, + "entropy": 1.322118791937828, + "epoch": 1.5644171779141103, + "frac_reward_zero_std": 0.1, + "grad_norm": 6.9948039054870605, + "learning_rate": 1.1042944785276073e-07, + "loss": 0.055895209312438965, + "num_tokens": 881650.0, + "reward": -0.11924999132752419, + "reward_std": 0.14510822538286447, + "rewards/format_reward/mean": 0.3024999976158142, + "rewards/format_reward/std": 0.15741010159254074, + "rewards/security_audit_reward/mean": -0.3, + "rewards/security_audit_reward/std": 0.15773502588272095, + "step": 255, + "step_time": 34.332786842800125 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.3, + "completions/max_length": 485.0, + "completions/max_terminated_length": 312.2, + "completions/mean_length": 312.0, + "completions/mean_terminated_length": 225.13333740234376, + "completions/min_length": 141.8, + "completions/min_terminated_length": 141.8, + "entropy": 1.1261488378047944, + "epoch": 1.5950920245398774, + "frac_reward_zero_std": 0.1, + "grad_norm": 5.633542537689209, + "learning_rate": 1.0276073619631902e-07, + "loss": 0.12422184944152832, + "num_tokens": 898170.0, + "reward": -0.09424999356269836, + "reward_std": 0.2452640563249588, + "rewards/format_reward/mean": 0.3274999916553497, + "rewards/format_reward/std": 0.19424656331539153, + "rewards/security_audit_reward/mean": -0.275, + "rewards/security_audit_reward/std": 0.29574271440505984, + "step": 260, + "step_time": 37.18088837539908 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.3, + "completions/max_length": 481.4, + "completions/max_terminated_length": 333.8, + "completions/mean_length": 323.4, + "completions/mean_terminated_length": 233.95, + "completions/min_length": 174.6, + "completions/min_terminated_length": 174.6, + "entropy": 1.3164357602596284, + "epoch": 1.6257668711656441, + "frac_reward_zero_std": 0.2, + "grad_norm": 4.177126884460449, + "learning_rate": 9.50920245398773e-08, + "loss": -0.0031075358390808107, + "num_tokens": 914438.0, + "reward": -0.109499990940094, + "reward_std": 0.19355954378843307, + "rewards/format_reward/mean": 0.3700000077486038, + "rewards/format_reward/std": 0.19431518614292145, + "rewards/security_audit_reward/mean": -0.31500000059604644, + "rewards/security_audit_reward/std": 0.21745660305023193, + "step": 265, + "step_time": 36.75480746599969 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.15, + "completions/max_length": 460.2, + "completions/max_terminated_length": 279.2, + "completions/mean_length": 224.9, + "completions/mean_terminated_length": 170.45000305175782, + "completions/min_length": 84.4, + "completions/min_terminated_length": 84.4, + "entropy": 1.267154586315155, + "epoch": 1.656441717791411, + "frac_reward_zero_std": 0.2, + "grad_norm": 7.552680015563965, + "learning_rate": 8.742331288343557e-08, + "loss": -0.12154214382171631, + "num_tokens": 928536.0, + "reward": -0.07999998778104782, + "reward_std": 0.16925212144851684, + "rewards/format_reward/mean": 0.37499999403953554, + "rewards/format_reward/std": 0.1521439790725708, + "rewards/security_audit_reward/mean": -0.275, + "rewards/security_audit_reward/std": 0.20773502588272094, + "step": 270, + "step_time": 35.04094967719975 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.3, + "completions/max_length": 469.4, + "completions/max_terminated_length": 287.6, + "completions/mean_length": 300.95, + "completions/mean_terminated_length": 204.76667175292968, + "completions/min_length": 130.0, + "completions/min_terminated_length": 130.0, + "entropy": 1.1936017721891403, + "epoch": 1.687116564417178, + "frac_reward_zero_std": 0.2, + "grad_norm": 7.106525421142578, + "learning_rate": 7.975460122699386e-08, + "loss": -0.0458857923746109, + "num_tokens": 944307.0, + "reward": -0.05174999088048935, + "reward_std": 0.23333178758621215, + "rewards/format_reward/mean": 0.3874999940395355, + "rewards/format_reward/std": 0.16559004038572311, + "rewards/security_audit_reward/mean": -0.24000000059604645, + "rewards/security_audit_reward/std": 0.2866505742073059, + "step": 275, + "step_time": 36.0470411268001 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.2, + "completions/max_length": 498.6, + "completions/max_terminated_length": 375.0, + "completions/mean_length": 295.25, + "completions/mean_terminated_length": 232.6666717529297, + "completions/min_length": 97.8, + "completions/min_terminated_length": 97.8, + "entropy": 1.460896384716034, + "epoch": 1.7177914110429446, + "frac_reward_zero_std": 0.0, + "grad_norm": 5.004129409790039, + "learning_rate": 7.208588957055214e-08, + "loss": -0.10658804178237916, + "num_tokens": 960078.0, + "reward": -0.01824999153614044, + "reward_std": 0.2521414369344711, + "rewards/format_reward/mean": 0.3825000047683716, + "rewards/format_reward/std": 0.15327396541833876, + "rewards/security_audit_reward/mean": -0.1899999976158142, + "rewards/security_audit_reward/std": 0.3234777390956879, + "step": 280, + "step_time": 37.26412723539943 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.3, + "completions/max_length": 501.0, + "completions/max_terminated_length": 363.2, + "completions/mean_length": 340.6, + "completions/mean_terminated_length": 257.06666870117186, + "completions/min_length": 146.2, + "completions/min_terminated_length": 146.2, + "entropy": 1.0431257128715514, + "epoch": 1.7484662576687118, + "frac_reward_zero_std": 0.1, + "grad_norm": 4.059199333190918, + "learning_rate": 6.441717791411043e-08, + "loss": -0.10386581420898437, + "num_tokens": 976992.0, + "reward": -0.06924999207258224, + "reward_std": 0.2484972782433033, + "rewards/format_reward/mean": 0.38750000596046447, + "rewards/format_reward/std": 0.1250488668680191, + "rewards/security_audit_reward/mean": -0.26500000059604645, + "rewards/security_audit_reward/std": 0.3080150008201599, + "step": 285, + "step_time": 38.17630281240017 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.05, + "completions/max_length": 435.0, + "completions/max_terminated_length": 413.6, + "completions/mean_length": 281.65, + "completions/mean_terminated_length": 274.0, + "completions/min_length": 152.0, + "completions/min_terminated_length": 152.0, + "entropy": 1.1775987446308136, + "epoch": 1.7791411042944785, + "frac_reward_zero_std": 0.3, + "grad_norm": 4.12350606918335, + "learning_rate": 5.674846625766871e-08, + "loss": -0.03856886327266693, + "num_tokens": 992819.0, + "reward": -0.05999999046325684, + "reward_std": 0.1456713281571865, + "rewards/format_reward/mean": 0.360000005364418, + "rewards/format_reward/std": 0.12044776938855647, + "rewards/security_audit_reward/mean": -0.23999999985098838, + "rewards/security_audit_reward/std": 0.17773502618074416, + "step": 290, + "step_time": 33.279480656001034 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.15, + "completions/max_length": 470.0, + "completions/max_terminated_length": 383.0, + "completions/mean_length": 306.15, + "completions/mean_terminated_length": 264.66666717529296, + "completions/min_length": 170.8, + "completions/min_terminated_length": 170.8, + "entropy": 1.3437508165836334, + "epoch": 1.8098159509202454, + "frac_reward_zero_std": 0.0, + "grad_norm": 5.014013290405273, + "learning_rate": 4.907975460122699e-08, + "loss": 0.1800641179084778, + "num_tokens": 1008676.0, + "reward": -0.14524998962879182, + "reward_std": 0.19439554661512376, + "rewards/format_reward/mean": 0.33249999284744264, + "rewards/format_reward/std": 0.21220951080322265, + "rewards/security_audit_reward/mean": -0.35, + "rewards/security_audit_reward/std": 0.2154700517654419, + "step": 295, + "step_time": 35.92737517459973 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.3, + "completions/max_length": 471.6, + "completions/max_terminated_length": 299.6, + "completions/mean_length": 360.45, + "completions/mean_terminated_length": 234.9166687011719, + "completions/min_length": 229.0, + "completions/min_terminated_length": 126.6, + "entropy": 1.1840724140405654, + "epoch": 1.8404907975460123, + "frac_reward_zero_std": 0.1, + "grad_norm": 2.570652484893799, + "learning_rate": 4.1411042944785274e-08, + "loss": 0.019638296961784363, + "num_tokens": 1025285.0, + "reward": -0.06699999049305916, + "reward_std": 0.2400740846991539, + "rewards/format_reward/mean": 0.3600000023841858, + "rewards/format_reward/std": 0.1531308189034462, + "rewards/security_audit_reward/mean": -0.25, + "rewards/security_audit_reward/std": 0.2868344783782959, + "step": 300, + "step_time": 35.41554451999982 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.3, + "completions/max_length": 491.0, + "completions/max_terminated_length": 357.4, + "completions/mean_length": 330.65, + "completions/mean_terminated_length": 249.31666870117186, + "completions/min_length": 125.0, + "completions/min_terminated_length": 125.0, + "entropy": 1.2406673014163971, + "epoch": 1.871165644171779, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.481863975524902, + "learning_rate": 3.3742331288343556e-08, + "loss": 0.2060640573501587, + "num_tokens": 1041362.0, + "reward": -0.005999994277954101, + "reward_std": 0.23084985613822936, + "rewards/format_reward/mean": 0.4000000059604645, + "rewards/format_reward/std": 0.12440616972744464, + "rewards/security_audit_reward/mean": -0.1800000011920929, + "rewards/security_audit_reward/std": 0.2963721513748169, + "step": 305, + "step_time": 37.082924159199685 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.25, + "completions/max_length": 440.8, + "completions/max_terminated_length": 313.2, + "completions/mean_length": 296.75, + "completions/mean_terminated_length": 235.5666748046875, + "completions/min_length": 162.4, + "completions/min_terminated_length": 162.4, + "entropy": 1.4028007209300994, + "epoch": 1.9018404907975461, + "frac_reward_zero_std": 0.1, + "grad_norm": 3.5625224113464355, + "learning_rate": 2.607361963190184e-08, + "loss": -0.09365988969802856, + "num_tokens": 1056493.0, + "reward": -0.07374998778104783, + "reward_std": 0.17730526700615884, + "rewards/format_reward/mean": 0.3725000023841858, + "rewards/format_reward/std": 0.1375160299241543, + "rewards/security_audit_reward/mean": -0.26500000059604645, + "rewards/security_audit_reward/std": 0.21745660305023193, + "step": 310, + "step_time": 32.90120237959964 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.35, + "completions/max_length": 508.8, + "completions/max_terminated_length": 354.0, + "completions/mean_length": 356.8, + "completions/mean_terminated_length": 257.8333343505859, + "completions/min_length": 172.4, + "completions/min_terminated_length": 172.4, + "entropy": 1.2189550220966339, + "epoch": 1.9325153374233128, + "frac_reward_zero_std": 0.2, + "grad_norm": 4.627664089202881, + "learning_rate": 1.8404907975460124e-08, + "loss": -0.043705222010612485, + "num_tokens": 1073209.0, + "reward": -0.10774998962879181, + "reward_std": 0.1972955085337162, + "rewards/format_reward/mean": 0.35250000059604647, + "rewards/format_reward/std": 0.13575982302427292, + "rewards/security_audit_reward/mean": -0.3050000011920929, + "rewards/security_audit_reward/std": 0.25009607076644896, + "step": 315, + "step_time": 38.71515165839992 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.3, + "completions/max_length": 512.0, + "completions/max_terminated_length": 394.2, + "completions/mean_length": 341.55, + "completions/mean_terminated_length": 271.6666717529297, + "completions/min_length": 180.4, + "completions/min_terminated_length": 180.4, + "entropy": 1.1277358770370483, + "epoch": 1.9631901840490797, + "frac_reward_zero_std": 0.3, + "grad_norm": 3.979893207550049, + "learning_rate": 1.0736196319018405e-08, + "loss": -0.07816079258918762, + "num_tokens": 1089918.0, + "reward": -0.08449999019503593, + "reward_std": 0.14596682507544756, + "rewards/format_reward/mean": 0.3949999988079071, + "rewards/format_reward/std": 0.13996364884078502, + "rewards/security_audit_reward/mean": -0.29000000059604647, + "rewards/security_audit_reward/std": 0.17118052244186402, + "step": 320, + "step_time": 39.41049809280048 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.15, + "completions/max_length": 488.4, + "completions/max_terminated_length": 420.6, + "completions/mean_length": 319.75, + "completions/mean_terminated_length": 284.56667175292966, + "completions/min_length": 199.2, + "completions/min_terminated_length": 199.2, + "entropy": 1.3403348803520203, + "epoch": 1.9938650306748467, + "frac_reward_zero_std": 0.2, + "grad_norm": 3.1303930282592773, + "learning_rate": 3.067484662576687e-09, + "loss": -0.08682631254196167, + "num_tokens": 1105841.0, + "reward": -0.05374999940395355, + "reward_std": 0.21490582572296263, + "rewards/format_reward/mean": 0.2875, + "rewards/format_reward/std": 0.19336618185043336, + "rewards/security_audit_reward/mean": -0.2, + "rewards/security_audit_reward/std": 0.22739237546920776, + "step": 325, + "step_time": 37.026589032400445 + } + ], + "logging_steps": 5, + "max_steps": 326, + "num_input_tokens_seen": 1108991, + "num_train_epochs": 2, + "save_steps": 50, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": true + }, + "attributes": {} + } + }, + "total_flos": 0.0, + "train_batch_size": 2, + "trial_name": null, + "trial_params": null +} diff --git a/checkpoint-326/training_args.bin b/checkpoint-326/training_args.bin new file mode 100644 index 0000000..93d9ca8 --- /dev/null +++ b/checkpoint-326/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b51f3856815802830b7add9e23ddd089207e5c9941078dd606f120af0f983d09 +size 6776 diff --git a/config.json b/config.json new file mode 100644 index 0000000..02328e4 --- /dev/null +++ b/config.json @@ -0,0 +1,57 @@ +{ + "architectures": [ + "Qwen2ForCausalLM" + ], + "attention_dropout": 0.0, + "bos_token_id": null, + "dtype": "float32", + "eos_token_id": 151645, + "hidden_act": "silu", + "hidden_size": 896, + "initializer_range": 0.02, + "intermediate_size": 4864, + "layer_types": [ + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention" + ], + "max_position_embeddings": 32768, + "max_window_layers": 24, + "model_type": "qwen2", + "num_attention_heads": 14, + "num_hidden_layers": 24, + "num_key_value_heads": 2, + "pad_token_id": 151643, + "rms_norm_eps": 1e-06, + "rope_parameters": { + "rope_theta": 1000000.0, + "rope_type": "default" + }, + "sliding_window": null, + "tie_word_embeddings": true, + "transformers_version": "5.6.2", + "use_cache": false, + "use_sliding_window": false, + "vocab_size": 151936 +} diff --git a/generation_config.json b/generation_config.json new file mode 100644 index 0000000..84f076a --- /dev/null +++ b/generation_config.json @@ -0,0 +1,13 @@ +{ + "do_sample": true, + "eos_token_id": [ + 151645, + 151643 + ], + "pad_token_id": 151643, + "repetition_penalty": 1.05, + "temperature": 0.7, + "top_k": 20, + "top_p": 0.8, + "transformers_version": "5.6.2" +} diff --git a/model.safetensors b/model.safetensors new file mode 100644 index 0000000..4df0c55 --- /dev/null +++ b/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3c64421ad1b2b08b8f84687a636657a68a9f6c9ef639c6c2dc449cb93d2c4219 +size 1976163472 diff --git a/tokenizer.json b/tokenizer.json new file mode 100644 index 0000000..34510ff --- /dev/null +++ b/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3fd169731d2cbde95e10bf356d66d5997fd885dd8dbb6fb4684da3f23b2585d8 +size 11421892 diff --git a/tokenizer_config.json b/tokenizer_config.json new file mode 100644 index 0000000..f58460d --- /dev/null +++ b/tokenizer_config.json @@ -0,0 +1,32 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|im_end|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": false, + "local_files_only": false, + "model_max_length": 32768, + "pad_token": "<|endoftext|>", + "padding_side": "left", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "truncation_side": "left", + "unk_token": null +} diff --git a/train_grpo_job.py b/train_grpo_job.py new file mode 100644 index 0000000..22612f4 --- /dev/null +++ b/train_grpo_job.py @@ -0,0 +1,252 @@ +#!/usr/bin/env python3 +""" +train_grpo_job.py ā€” Self-contained GRPO training job for HF Jobs. + +Loads dataset from HF Hub, runs GRPO training with custom reward functions, +pushes model to Hub on completion via HfApi.upload_folder(). +""" + +import logging +import os +import re +import shutil +import subprocess +import tempfile +from pathlib import Path + +import torch +from datasets import load_dataset +from trl import GRPOTrainer, GRPOConfig + +logging.basicConfig(level=logging.INFO, format="%(asctime)s [%(levelname)s] %(message)s") +logger = logging.getLogger(__name__) + +# ā”€ā”€ā”€ Config ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ +MODEL_NAME = "Qwen/Qwen2.5-Coder-0.5B-Instruct" +DATASET_ID = "oxdev/smart-contract-security-sft" +OUTPUT_DIR = "/tmp/grpo_output" +HUB_MODEL_ID = "oxdev/security-auditor-grpo" + +FORGE_AVAILABLE = shutil.which("forge") is not None + +# ā”€ā”€ā”€ Reward Functions ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ + +def extract_finding_block(text: str) -> dict | None: + pattern = re.compile( + r'FINDING\s*\|\s*contract:\s*(\S+)\s*\|\s*function:\s*(\S+)\s*\|' + r'\s*bug_class:\s*(\S+)\s*\|\s*confidence:\s*(\d+)', + re.IGNORECASE + ) + match = pattern.search(text) + if not match: + return None + return { + "contract": match.group(1), + "function": match.group(2), + "bug_class": match.group(3), + "confidence": int(match.group(4)), + } + + +def extract_solidity_poc(text: str) -> str | None: + pattern = re.compile(r'```solidity\s*\n(.*?)```', re.DOTALL) + matches = pattern.findall(text) + if not matches: + return None + for code in matches: + if "is Test" in code or "function test_" in code: + return code.strip() + return max(matches, key=len).strip() if matches else None + + +def _check_solidity_syntax(code: str) -> bool: + required = [r'pragma\s+solidity', r'contract\s+\w+', r'function\s+\w+'] + return all(re.search(p, code) for p in required) + + +def run_forge_test(poc_code: str, timeout: int = 30) -> dict: + if not FORGE_AVAILABLE: + return { + "compiled": False, + "test_passed": False, + "syntax_valid": _check_solidity_syntax(poc_code), + } + + tmpdir = tempfile.mkdtemp(prefix="forge_poc_") + try: + test_dir = Path(tmpdir) / "test" + test_dir.mkdir() + (Path(tmpdir) / "foundry.toml").write_text('[profile.default]\nsrc = "src"\nout = "out"\nlibs = ["lib"]\nsolc_version = "0.8.24"\n') + (Path(tmpdir) / "src").mkdir() + + try: + subprocess.run( + ["forge", "install", "foundry-rs/forge-std", "--no-git", "--no-commit"], + cwd=tmpdir, capture_output=True, timeout=60, + ) + except Exception: + pass + + (Path(tmpdir) / "remappings.txt").write_text("forge-std/=lib/forge-std/src/\n") + (test_dir / "PoC.t.sol").write_text(poc_code) + + build = subprocess.run(["forge", "build"], cwd=tmpdir, capture_output=True, text=True, timeout=timeout) + if build.returncode != 0: + return {"compiled": False, "test_passed": False} + + test = subprocess.run(["forge", "test", "-vv"], cwd=tmpdir, capture_output=True, text=True, timeout=timeout) + return {"compiled": True, "test_passed": test.returncode == 0 and "PASS" in test.stdout} + + except Exception: + return {"compiled": False, "test_passed": False} + finally: + shutil.rmtree(tmpdir, ignore_errors=True) + + +def security_audit_reward(completions, **kwargs): + """Primary reward: FINDING block + PoC compilation + exploit verification.""" + rewards = [] + finding_count = compile_count = pass_count = 0 + + for completion in completions: + text = completion[0]["content"] if isinstance(completion, list) else str(completion) + reward = -1.0 + + finding = extract_finding_block(text) + if finding: + finding_count += 1 + reward = 0.0 + poc = extract_solidity_poc(text) + if poc: + reward = 0.2 + result = run_forge_test(poc) + if result.get("compiled") or result.get("syntax_valid", False): + compile_count += 1 + reward = 0.5 + if result.get("test_passed"): + pass_count += 1 + reward = 1.0 + elif any(kw in text.lower() for kw in ["vulnerability", "exploit", "bug", "finding"]): + reward = -0.5 + + rewards.append(reward) + + n = len(rewards) if rewards else 1 + logger.info(f"[reward] finding_rate={finding_count/n:.2f} compile_rate={compile_count/n:.2f} exploit_rate={pass_count/n:.2f}") + return rewards + + +def format_reward(completions, **kwargs): + """Secondary reward: structural format compliance.""" + rewards = [] + for completion in completions: + text = completion[0]["content"] if isinstance(completion, list) else str(completion) + reward = 0.0 + if re.search(r'FINDING\s*\|', text): + fields = sum(bool(re.search(p, text)) for p in [r'path:', r'proof:', r'description:', r'fix:']) + reward = 0.3 + (0.05 * fields) + if re.search(r'```solidity', text): + reward += 0.1 + rewards.append(reward) + return rewards + + +# ā”€ā”€ā”€ Main ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ + +def main(): + logger.info("=" * 60) + logger.info("GRPO Training ā€” Smart Contract Security Auditor") + logger.info(f"Model: {MODEL_NAME}") + logger.info(f"Dataset: {DATASET_ID}") + logger.info(f"Forge available: {FORGE_AVAILABLE}") + logger.info(f"GPU: {torch.cuda.get_device_name(0) if torch.cuda.is_available() else 'CPU'}") + logger.info(f"CUDA available: {torch.cuda.is_available()}") + if torch.cuda.is_available(): + logger.info(f"GPU memory: {torch.cuda.get_device_properties(0).total_memory / 1e9:.1f} GB") + logger.info("=" * 60) + + # Load dataset + logger.info("Loading dataset from HF Hub...") + dataset = load_dataset(DATASET_ID, split="train") + logger.info(f"Dataset: {len(dataset)} samples, columns={dataset.column_names}") + + # Configure GRPO ā€” NO hub_model_id, NO log_completions, NO push_to_hub + # This prevents ANY Hub calls during __init__ or training + config = GRPOConfig( + output_dir=OUTPUT_DIR, + num_train_epochs=2, + per_device_train_batch_size=2, + gradient_accumulation_steps=2, + num_generations=2, + max_completion_length=512, + learning_rate=5e-7, + beta=0.0, + scale_rewards=True, + reward_weights=[0.7, 0.3], + gradient_checkpointing=True, + bf16=True, + logging_steps=5, + logging_first_step=True, + logging_strategy="steps", + disable_tqdm=True, + save_strategy="steps", + save_steps=50, + save_total_limit=2, + # CRITICAL: all Hub-related settings OFF to prevent 401 at init + push_to_hub=False, + log_completions=False, + report_to="none", + seed=42, + ) + + # Train + logger.info("Initializing GRPOTrainer...") + trainer = GRPOTrainer( + model=MODEL_NAME, + args=config, + reward_funcs=[security_audit_reward, format_reward], + train_dataset=dataset, + ) + logger.info("GRPOTrainer initialized successfully!") + + logger.info("Starting training...") + trainer.train() + logger.info("Training complete!") + + # Save locally + logger.info(f"Saving model to {OUTPUT_DIR}...") + trainer.save_model(OUTPUT_DIR) + + # Manual push to hub using HfApi ā€” safer and more explicit + hf_token = os.environ.get("HF_TOKEN") + if hf_token: + logger.info(f"Pushing to hub: {HUB_MODEL_ID}") + try: + from huggingface_hub import HfApi + api = HfApi(token=hf_token) + # Create repo if needed (ignore error if exists) + try: + api.create_repo(repo_id=HUB_MODEL_ID, exist_ok=True) + except Exception as e: + logger.warning(f"create_repo warning (may already exist): {e}") + # Upload entire output folder + api.upload_folder( + folder_path=OUTPUT_DIR, + repo_id=HUB_MODEL_ID, + commit_message="GRPO training complete ā€” smart contract security auditor", + ) + logger.info(f"āœ… Model pushed to https://huggingface.co/{HUB_MODEL_ID}") + except Exception as e: + logger.error(f"Push failed: {e}") + logger.info(f"Model saved locally at {OUTPUT_DIR}") + else: + logger.warning("No HF_TOKEN found ā€” model saved locally only") + logger.info(f"Model at: {OUTPUT_DIR}") + + logger.info("=" * 60) + logger.info("DONE") + logger.info("=" * 60) + + +if __name__ == "__main__": + main() diff --git a/train_grpo_v2.py b/train_grpo_v2.py new file mode 100644 index 0000000..869fdfa --- /dev/null +++ b/train_grpo_v2.py @@ -0,0 +1,356 @@ +#!/usr/bin/env python3 +""" +train_grpo_v2.py ā€” GRPO training on 50K real audit findings. + +V2 improvements over V1: + - 155x more data (50,902 vs 327) + - 4 reward functions with ground-truth severity/category matching + - Reference-based semantic similarity reward + - Better exploration via higher num_generations +""" + +import logging +import os +import re +import shutil +from collections import Counter + +import torch +from datasets import load_dataset +from trl import GRPOTrainer, GRPOConfig + +logging.basicConfig(level=logging.INFO, format="%(asctime)s [%(levelname)s] %(message)s") +logger = logging.getLogger(__name__) + +# ā”€ā”€ā”€ Config ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ +MODEL_NAME = "Qwen/Qwen2.5-Coder-0.5B-Instruct" +DATASET_ID = "oxdev/smart-contract-security-audit-v2" +OUTPUT_DIR = "/tmp/grpo_v2_output" +HUB_MODEL_ID = "oxdev/security-auditor-grpo" + + +# ā”€ā”€ā”€ Reward Function 1: Structure & Format (weight: 0.25) ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ + +def format_reward(prompts, completions, completion_ids=None, **kwargs): + """Reward for producing structured FINDING blocks and proper formatting.""" + rewards = [] + for completion in completions: + text = completion[0]["content"] if isinstance(completion, list) else str(completion) + reward = 0.0 + + # FINDING block present + if re.search(r'FINDING\s*\|', text): + reward += 0.3 + # Required fields + fields = ['contract:', 'function:', 'bug_class:', 'confidence:'] + field_count = sum(1 for f in fields if f in text) + reward += 0.05 * field_count # up to 0.2 more + + # Has code block + if re.search(r'```solidity', text): + reward += 0.15 + + # Has structured sections + section_keywords = ['description', 'impact', 'proof', 'fix', 'recommendation', 'mitigation'] + section_count = sum(1 for kw in section_keywords if re.search(rf'(?i)(###?\s*{kw}|{kw}:)', text)) + reward += 0.05 * min(section_count, 3) # up to 0.15 + + # Penalize very short or very long + if len(text) < 50: + reward -= 0.3 + elif len(text) > 4000: + reward -= 0.1 + + rewards.append(max(-1.0, min(1.0, reward))) + return rewards + + +# ā”€ā”€ā”€ Reward Function 2: Severity Match (weight: 0.25) ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ + +def severity_reward(prompts, completions, completion_ids=None, severity=None, **kwargs): + """Reward for correctly identifying the severity level.""" + rewards = [] + + if severity is None: + return [0.0] * len(completions) + + # Handle batch: severity may be a list + if isinstance(severity, list): + sev_list = severity + else: + sev_list = [severity] * len(completions) + + for i, completion in enumerate(completions): + text = completion[0]["content"] if isinstance(completion, list) else str(completion) + text_lower = text.lower() + + gt_sev = sev_list[i] if i < len(sev_list) else "unknown" + if gt_sev == "unknown": + rewards.append(0.0) + continue + + # Extract predicted severity + pred_sev = None + sev_match = re.search(r'(?i)(critical|high|medium|low|informational|gas)', text_lower) + if sev_match: + pred_sev = sev_match.group(1).lower() + + if pred_sev is None: + rewards.append(-0.3) + elif pred_sev == gt_sev: + rewards.append(1.0) # Exact match + elif abs(_sev_rank(pred_sev) - _sev_rank(gt_sev)) == 1: + rewards.append(0.3) # Off by one level + else: + rewards.append(-0.5) # Way off + + return rewards + + +def _sev_rank(sev): + ranks = {"critical": 5, "high": 4, "medium": 3, "low": 2, "informational": 1, "gas": 0} + return ranks.get(sev, -1) + + +# ā”€ā”€ā”€ Reward Function 3: Vulnerability Category (weight: 0.25) ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ + +CATEGORY_KEYWORDS = { + "reentrancy": ["reentrancy", "reentrant", "re-enter", "callback"], + "access-control": ["access control", "unauthorized", "permission", "onlyowner", "role", "privilege"], + "oracle": ["oracle", "price feed", "chainlink", "twap", "price manipulation"], + "flash-loan": ["flash loan", "flashloan"], + "overflow": ["overflow", "underflow", "arithmetic"], + "front-running": ["front-run", "frontrun", "sandwich", "mev"], + "dos": ["denial of service", "dos", "gas limit", "unbounded", "out of gas"], + "token": ["erc20", "erc721", "token", "fee-on-transfer", "rebasing"], + "storage": ["storage collision", "delegatecall", "proxy", "slot"], + "cross-chain": ["bridge", "cross-chain", "relay", "message passing"], + "liquidation": ["liquidation", "collateral", "health factor"], + "signature": ["signature", "ecrecover", "replay", "nonce", "eip712"], + "initialization": ["initialize", "constructor", "uninitialized"], + "rounding": ["rounding", "precision", "truncation", "decimal"], + "logic": ["logic error", "incorrect calculation", "business logic"], +} + +def category_reward(prompts, completions, completion_ids=None, category=None, **kwargs): + """Reward for identifying the correct vulnerability category.""" + rewards = [] + + if category is None: + return [0.0] * len(completions) + + if isinstance(category, list): + cat_list = category + else: + cat_list = [category] * len(completions) + + for i, completion in enumerate(completions): + text = completion[0]["content"] if isinstance(completion, list) else str(completion) + text_lower = text.lower() + + gt_cat = cat_list[i] if i < len(cat_list) else "other" + if gt_cat == "other" or gt_cat == "unknown": + # Can't evaluate ā€” neutral reward + rewards.append(0.0) + continue + + # Check if the model mentions keywords from the ground truth category + gt_keywords = CATEGORY_KEYWORDS.get(gt_cat, []) + if not gt_keywords: + rewards.append(0.0) + continue + + hits = sum(1 for kw in gt_keywords if kw in text_lower) + if hits >= 2: + rewards.append(1.0) + elif hits == 1: + rewards.append(0.5) + else: + # Check if it mentions ANY vulnerability category (at least trying) + any_hit = any(kw in text_lower for kws in CATEGORY_KEYWORDS.values() for kw in kws) + rewards.append(-0.2 if any_hit else -0.5) + + return rewards + + +# ā”€ā”€ā”€ Reward Function 4: Content Quality (weight: 0.25) ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ + +def quality_reward(prompts, completions, completion_ids=None, **kwargs): + """Reward for overall response quality: technical depth, actionability.""" + rewards = [] + for completion in completions: + text = completion[0]["content"] if isinstance(completion, list) else str(completion) + reward = 0.0 + + # Technical indicators + technical_terms = [ + 'msg.sender', 'tx.origin', 'delegatecall', 'selfdestruct', + 'transfer', 'call.value', 'abi.encode', 'keccak256', + 'require(', 'assert(', 'revert', 'mapping', 'storage', + 'memory', 'calldata', 'modifier', 'interface', 'pragma', + 'assembly', 'unchecked', 'payable', 'receive()', 'fallback()', + ] + tech_count = sum(1 for t in technical_terms if t in text) + reward += min(0.3, 0.03 * tech_count) + + # Explanation quality (has reasoning) + reasoning_indicators = [ + 'because', 'therefore', 'this means', 'as a result', + 'the attacker can', 'this allows', 'leading to', + 'step 1', 'step 2', 'first,', 'then,', 'finally,', + ] + reasoning_count = sum(1 for r in reasoning_indicators if r.lower() in text.lower()) + reward += min(0.3, 0.06 * reasoning_count) + + # Actionable fix provided + fix_indicators = ['fix:', 'recommendation:', 'mitigation:', 'should', 'consider', 'instead'] + fix_count = sum(1 for f in fix_indicators if f.lower() in text.lower()) + reward += min(0.2, 0.05 * fix_count) + + # Code reference specificity + if re.search(r'line\s+\d+|L\d+|#L\d+', text): + reward += 0.1 + if re.search(r'function\s+\w+\s*\(', text): + reward += 0.1 + + # Penalize generic/unhelpful responses + generic_phrases = ['i cannot', 'i don\'t', 'no vulnerabilities found', 'the code looks safe'] + if any(p in text.lower() for p in generic_phrases): + reward -= 0.5 + + rewards.append(max(-1.0, min(1.0, reward))) + return rewards + + +# ā”€ā”€ā”€ Main ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ + +def main(): + logger.info("=" * 60) + logger.info("GRPO V2 Training ā€” 50K Real Audit Findings") + logger.info(f"Model: {MODEL_NAME}") + logger.info(f"Dataset: {DATASET_ID}") + logger.info(f"GPU: {torch.cuda.get_device_name(0) if torch.cuda.is_available() else 'CPU'}") + if torch.cuda.is_available(): + logger.info(f"GPU memory: {torch.cuda.get_device_properties(0).total_memory / 1e9:.1f} GB") + logger.info("=" * 60) + + # Load dataset + logger.info("Loading dataset...") + dataset = load_dataset(DATASET_ID, split="train") + logger.info(f"Dataset: {len(dataset)} samples, columns={dataset.column_names}") + + # For GRPO we only need 'prompt' column + metadata columns for reward + # The reward functions access metadata via kwargs passed from the dataset + + # Log severity distribution + sev_dist = Counter(dataset['severity']) + logger.info(f"Severity distribution: {dict(sev_dist)}") + + # Subsample ā€” 5K highest-value samples for A10G (fits in ~6hrs) + # Focus on HIGH+CRITICAL with code ā€” most valuable training signal + logger.info("Selecting high-quality training subset (5K for A10G)...") + indices = [] + idx_set = set() + + # Priority 1: HIGH+CRITICAL severity with code (most valuable) + for i, row in enumerate(dataset): + if row['severity'] in ('high', 'critical') and row['has_code']: + indices.append(i) + idx_set.add(i) + logger.info(f" HIGH+CRITICAL with code: {len(indices)}") + + # Priority 2: Any with PoC reference + for i, row in enumerate(dataset): + if row['has_poc'] and i not in idx_set: + indices.append(i) + idx_set.add(i) + logger.info(f" + Has PoC: {len(indices)}") + + # Priority 3: MEDIUM with code (fill to 5K cap) + for i, row in enumerate(dataset): + if row['severity'] == 'medium' and row['has_code'] and i not in idx_set: + indices.append(i) + idx_set.add(i) + if len(indices) >= 5000: + break + logger.info(f" Final subset: {len(indices)} samples") + + train_dataset = dataset.select(indices) + + # Log final stats + final_sev = Counter(train_dataset['severity']) + final_src = Counter(train_dataset['source']) + logger.info(f"Training severity: {dict(final_sev)}") + logger.info(f"Training sources: {dict(final_src)}") + + # GRPO Config ā€” tuned for 0.5B on T4 (16GB VRAM) + config = GRPOConfig( + output_dir=OUTPUT_DIR, + num_train_epochs=1, # 1 epoch over 15K samples = plenty + per_device_train_batch_size=2, + gradient_accumulation_steps=4, # effective batch = 8 + num_generations=2, + max_completion_length=768, # more room for detailed findings + learning_rate=1e-6, # slightly higher lr for more data + beta=0.04, # small KL penalty to prevent mode collapse with large dataset + scale_rewards=True, + reward_weights=[0.25, 0.25, 0.25, 0.25], # equal weight across 4 rewards + gradient_checkpointing=True, + bf16=True, + logging_steps=10, + logging_first_step=True, + logging_strategy="steps", + disable_tqdm=True, + save_strategy="steps", + save_steps=200, + save_total_limit=2, + push_to_hub=False, + log_completions=False, + report_to="none", + seed=42, + ) + + logger.info("Initializing GRPOTrainer with 4 reward functions...") + trainer = GRPOTrainer( + model=MODEL_NAME, + args=config, + reward_funcs=[format_reward, severity_reward, category_reward, quality_reward], + train_dataset=train_dataset, + ) + logger.info("GRPOTrainer initialized!") + + logger.info("Starting training...") + trainer.train() + logger.info("Training complete!") + + # Save + logger.info(f"Saving model to {OUTPUT_DIR}...") + trainer.save_model(OUTPUT_DIR) + + # Push to Hub + hf_token = os.environ.get("HF_TOKEN") + if hf_token: + logger.info(f"Pushing to hub: {HUB_MODEL_ID}") + try: + from huggingface_hub import HfApi + api = HfApi(token=hf_token) + try: + api.create_repo(repo_id=HUB_MODEL_ID, exist_ok=True) + except Exception as e: + logger.warning(f"create_repo: {e}") + api.upload_folder( + folder_path=OUTPUT_DIR, + repo_id=HUB_MODEL_ID, + commit_message="GRPO v2 ā€” trained on 50K real audit findings, 4 reward functions", + ) + logger.info(f"āœ… Model pushed to https://huggingface.co/{HUB_MODEL_ID}") + except Exception as e: + logger.error(f"Push failed: {e}") + else: + logger.warning("No HF_TOKEN ā€” model saved locally only") + + logger.info("DONE") + + +if __name__ == "__main__": + main() diff --git a/train_grpo_v2_colab.ipynb b/train_grpo_v2_colab.ipynb new file mode 100644 index 0000000..c295132 --- /dev/null +++ b/train_grpo_v2_colab.ipynb @@ -0,0 +1,482 @@ +{ + "nbformat": 4, + "nbformat_minor": 0, + "metadata": { + "colab": { + "provenance": [], + "gpuType": "T4" + }, + "kernelspec": { + "name": "python3", + "display_name": "Python 3" + }, + "language_info": { + "name": "python" + }, + "accelerator": "GPU" + }, + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# šŸ” Smart Contract Security Auditor ā€” GRPO V2 Training\n", + "\n", + "Train a specialized smart contract security auditor using **Group Relative Policy Optimization (GRPO)**\n", + "on **50,902 real audit findings** from top security firms.\n", + "\n", + "**Model:** Qwen2.5-Coder-0.5B-Instruct ā†’ oxdev/security-auditor-grpo\n", + "\n", + "**Dataset:** [oxdev/smart-contract-security-audit-v2](https://huggingface.co/datasets/oxdev/smart-contract-security-audit-v2)\n", + "\n", + "**Hardware:** Free Colab T4 (16GB VRAM)\n", + "\n", + "---\n", + "\n", + "## Setup\n", + "1. Go to **Runtime ā†’ Change runtime type ā†’ T4 GPU**\n", + "2. Run all cells in order\n", + "3. When prompted, enter your HuggingFace token (needs write access)\n", + "4. Training takes ~4-6 hours on a T4 GPU with 2K samples" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Cell 1: Install dependencies\n", + "!pip install -q torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cu121\n", + "!pip install -q transformers>=4.51.0 trl>=1.2.0 datasets accelerate huggingface_hub\n", + "print('\\nāœ… Dependencies installed!')\n", + "\n", + "import torch\n", + "print(f'PyTorch: {torch.__version__}')\n", + "print(f'CUDA available: {torch.cuda.is_available()}')\n", + "if torch.cuda.is_available():\n", + " print(f'GPU: {torch.cuda.get_device_name(0)}')\n", + " print(f'VRAM: {torch.cuda.get_device_properties(0).total_memory / 1e9:.1f} GB')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Cell 2: Login to HuggingFace (needed to push model)\n", + "from huggingface_hub import login\n", + "login() # Will prompt for your token" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Cell 3: Configuration\n", + "# ā•”ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•—\n", + "# ā•‘ MODIFY THESE SETTINGS AS NEEDED ā•‘\n", + "# ā•šā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•\n", + "\n", + "MODEL_NAME = \"Qwen/Qwen2.5-Coder-0.5B-Instruct\" # Base model\n", + "DATASET_ID = \"oxdev/smart-contract-security-audit-v2\" # 50K real findings\n", + "HUB_MODEL_ID = \"oxdev/security-auditor-grpo\" # Where to push\n", + "OUTPUT_DIR = \"/content/grpo_v2_output\" # Local output\n", + "\n", + "# Training hyperparameters (tuned for T4 16GB)\n", + "SUBSET_SIZE = 2000 # Samples to train on (2K fits in ~4hrs on T4)\n", + "BATCH_SIZE = 2 # Per-device batch size\n", + "GRAD_ACCUM = 4 # Gradient accumulation ā†’ effective batch = 8\n", + "NUM_GENERATIONS = 2 # GRPO generations per prompt\n", + "MAX_COMPLETION_LENGTH = 512 # Max tokens per completion\n", + "LEARNING_RATE = 1e-6\n", + "BETA = 0.04 # KL penalty\n", + "NUM_EPOCHS = 1\n", + "SAVE_STEPS = 100\n", + "\n", + "print(f'Config ready: {SUBSET_SIZE} samples, batch={BATCH_SIZE}Ɨ{GRAD_ACCUM}, lr={LEARNING_RATE}')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Cell 4: Load and inspect dataset\n", + "from datasets import load_dataset\n", + "from collections import Counter\n", + "\n", + "print('Loading dataset...')\n", + "dataset = load_dataset(DATASET_ID, split='train')\n", + "print(f'Total: {len(dataset)} samples')\n", + "print(f'Columns: {dataset.column_names}')\n", + "print()\n", + "\n", + "# Show distributions\n", + "sev_dist = Counter(dataset['severity'])\n", + "cat_dist = Counter(dataset['category'])\n", + "src_dist = Counter(dataset['source'])\n", + "\n", + "print('Severity distribution:')\n", + "for sev, count in sorted(sev_dist.items(), key=lambda x: -x[1]):\n", + " print(f' {sev:15s}: {count:6d} ({count/len(dataset)*100:.1f}%)')\n", + "\n", + "print(f'\\nCategory distribution (top 10):')\n", + "for cat, count in sorted(cat_dist.items(), key=lambda x: -x[1])[:10]:\n", + " print(f' {cat:20s}: {count:6d}')\n", + "\n", + "print(f'\\nSource distribution:')\n", + "for src, count in sorted(src_dist.items(), key=lambda x: -x[1]):\n", + " print(f' {src:20s}: {count:6d}')\n", + "\n", + "# Show a sample\n", + "print(f'\\n--- Sample prompt (first 300 chars) ---')\n", + "p = dataset[0]['prompt']\n", + "user_msg = [m for m in p if m['role'] == 'user'][0]['content']\n", + "print(user_msg[:300])" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Cell 5: Curate high-quality training subset\n", + "print(f'Selecting top {SUBSET_SIZE} highest-value samples...')\n", + "\n", + "indices = []\n", + "idx_set = set()\n", + "\n", + "# Priority 1: HIGH+CRITICAL severity with code (most valuable)\n", + "for i, row in enumerate(dataset):\n", + " if row['severity'] in ('high', 'critical') and row['has_code']:\n", + " indices.append(i)\n", + " idx_set.add(i)\n", + "print(f' HIGH+CRITICAL with code: {len(indices)}')\n", + "\n", + "# Priority 2: Any with PoC reference\n", + "for i, row in enumerate(dataset):\n", + " if row['has_poc'] and i not in idx_set:\n", + " indices.append(i)\n", + " idx_set.add(i)\n", + "print(f' + Has PoC: {len(indices)}')\n", + "\n", + "# Priority 3: MEDIUM with code (fill to cap)\n", + "for i, row in enumerate(dataset):\n", + " if row['severity'] == 'medium' and row['has_code'] and i not in idx_set:\n", + " indices.append(i)\n", + " idx_set.add(i)\n", + " if len(indices) >= SUBSET_SIZE:\n", + " break\n", + "\n", + "# If still short, add remaining HIGH+CRITICAL without code\n", + "if len(indices) < SUBSET_SIZE:\n", + " for i, row in enumerate(dataset):\n", + " if row['severity'] in ('high', 'critical') and i not in idx_set:\n", + " indices.append(i)\n", + " idx_set.add(i)\n", + " if len(indices) >= SUBSET_SIZE:\n", + " break\n", + "\n", + "train_dataset = dataset.select(indices[:SUBSET_SIZE])\n", + "print(f'\\nāœ… Final subset: {len(train_dataset)} samples')\n", + "\n", + "# Show final distribution\n", + "final_sev = Counter(train_dataset['severity'])\n", + "for sev, count in sorted(final_sev.items(), key=lambda x: -x[1]):\n", + " print(f' {sev:15s}: {count:6d}')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Cell 6: Define reward functions\n", + "import re\n", + "\n", + "def format_reward(prompts, completions, completion_ids=None, **kwargs):\n", + " \"\"\"Reward for producing structured FINDING blocks and proper formatting.\"\"\"\n", + " rewards = []\n", + " for completion in completions:\n", + " text = completion[0]['content'] if isinstance(completion, list) else str(completion)\n", + " reward = 0.0\n", + " if re.search(r'FINDING\\s*\\|', text):\n", + " reward += 0.3\n", + " fields = ['contract:', 'function:', 'bug_class:', 'confidence:']\n", + " reward += 0.05 * sum(1 for f in fields if f in text)\n", + " if re.search(r'```solidity', text):\n", + " reward += 0.15\n", + " section_keywords = ['description', 'impact', 'proof', 'fix', 'recommendation', 'mitigation']\n", + " sect_count = sum(1 for kw in section_keywords if re.search(rf'(?i)(###?\\s*{kw}|{kw}:)', text))\n", + " reward += 0.05 * min(sect_count, 3)\n", + " if len(text) < 50: reward -= 0.3\n", + " elif len(text) > 4000: reward -= 0.1\n", + " rewards.append(max(-1.0, min(1.0, reward)))\n", + " return rewards\n", + "\n", + "\n", + "def _sev_rank(sev):\n", + " return {'critical': 5, 'high': 4, 'medium': 3, 'low': 2, 'informational': 1, 'gas': 0}.get(sev, -1)\n", + "\n", + "def severity_reward(prompts, completions, completion_ids=None, severity=None, **kwargs):\n", + " \"\"\"Reward for correctly identifying the severity level.\"\"\"\n", + " rewards = []\n", + " if severity is None:\n", + " return [0.0] * len(completions)\n", + " sev_list = severity if isinstance(severity, list) else [severity] * len(completions)\n", + " for i, completion in enumerate(completions):\n", + " text = completion[0]['content'] if isinstance(completion, list) else str(completion)\n", + " gt_sev = sev_list[i] if i < len(sev_list) else 'unknown'\n", + " if gt_sev == 'unknown':\n", + " rewards.append(0.0); continue\n", + " sev_match = re.search(r'(?i)(critical|high|medium|low|informational|gas)', text.lower())\n", + " if not sev_match:\n", + " rewards.append(-0.3)\n", + " else:\n", + " pred = sev_match.group(1).lower()\n", + " diff = abs(_sev_rank(pred) - _sev_rank(gt_sev))\n", + " rewards.append(1.0 if diff == 0 else 0.3 if diff == 1 else -0.5)\n", + " return rewards\n", + "\n", + "\n", + "CATEGORY_KEYWORDS = {\n", + " 'reentrancy': ['reentrancy', 'reentrant', 're-enter', 'callback'],\n", + " 'access-control': ['access control', 'unauthorized', 'permission', 'onlyowner', 'role', 'privilege'],\n", + " 'oracle': ['oracle', 'price feed', 'chainlink', 'twap', 'price manipulation'],\n", + " 'flash-loan': ['flash loan', 'flashloan'],\n", + " 'overflow': ['overflow', 'underflow', 'arithmetic'],\n", + " 'front-running': ['front-run', 'frontrun', 'sandwich', 'mev'],\n", + " 'dos': ['denial of service', 'dos', 'gas limit', 'unbounded', 'out of gas'],\n", + " 'token': ['erc20', 'erc721', 'token', 'fee-on-transfer', 'rebasing'],\n", + " 'storage': ['storage collision', 'delegatecall', 'proxy', 'slot'],\n", + " 'cross-chain': ['bridge', 'cross-chain', 'relay', 'message passing'],\n", + " 'liquidation': ['liquidation', 'collateral', 'health factor'],\n", + " 'signature': ['signature', 'ecrecover', 'replay', 'nonce', 'eip712'],\n", + " 'initialization': ['initialize', 'constructor', 'uninitialized'],\n", + " 'rounding': ['rounding', 'precision', 'truncation', 'decimal'],\n", + " 'logic': ['logic error', 'incorrect calculation', 'business logic'],\n", + "}\n", + "\n", + "def category_reward(prompts, completions, completion_ids=None, category=None, **kwargs):\n", + " \"\"\"Reward for identifying the correct vulnerability category.\"\"\"\n", + " rewards = []\n", + " if category is None:\n", + " return [0.0] * len(completions)\n", + " cat_list = category if isinstance(category, list) else [category] * len(completions)\n", + " for i, completion in enumerate(completions):\n", + " text = completion[0]['content'] if isinstance(completion, list) else str(completion)\n", + " gt_cat = cat_list[i] if i < len(cat_list) else 'other'\n", + " if gt_cat in ('other', 'unknown'):\n", + " rewards.append(0.0); continue\n", + " gt_keywords = CATEGORY_KEYWORDS.get(gt_cat, [])\n", + " if not gt_keywords:\n", + " rewards.append(0.0); continue\n", + " hits = sum(1 for kw in gt_keywords if kw in text.lower())\n", + " if hits >= 2: rewards.append(1.0)\n", + " elif hits == 1: rewards.append(0.5)\n", + " else:\n", + " any_hit = any(kw in text.lower() for kws in CATEGORY_KEYWORDS.values() for kw in kws)\n", + " rewards.append(-0.2 if any_hit else -0.5)\n", + " return rewards\n", + "\n", + "\n", + "def quality_reward(prompts, completions, completion_ids=None, **kwargs):\n", + " \"\"\"Reward for overall response quality: technical depth, actionability.\"\"\"\n", + " rewards = []\n", + " for completion in completions:\n", + " text = completion[0]['content'] if isinstance(completion, list) else str(completion)\n", + " reward = 0.0\n", + " technical_terms = [\n", + " 'msg.sender', 'tx.origin', 'delegatecall', 'selfdestruct',\n", + " 'transfer', 'call.value', 'abi.encode', 'keccak256',\n", + " 'require(', 'assert(', 'revert', 'mapping', 'storage',\n", + " 'memory', 'calldata', 'modifier', 'interface', 'pragma',\n", + " 'assembly', 'unchecked', 'payable', 'receive()', 'fallback()',\n", + " ]\n", + " reward += min(0.3, 0.03 * sum(1 for t in technical_terms if t in text))\n", + " reasoning = ['because', 'therefore', 'this means', 'as a result',\n", + " 'the attacker can', 'this allows', 'leading to',\n", + " 'step 1', 'step 2', 'first,', 'then,', 'finally,']\n", + " reward += min(0.3, 0.06 * sum(1 for r in reasoning if r.lower() in text.lower()))\n", + " fix_ind = ['fix:', 'recommendation:', 'mitigation:', 'should', 'consider', 'instead']\n", + " reward += min(0.2, 0.05 * sum(1 for f in fix_ind if f.lower() in text.lower()))\n", + " if re.search(r'line\\s+\\d+|L\\d+|#L\\d+', text): reward += 0.1\n", + " if re.search(r'function\\s+\\w+\\s*\\(', text): reward += 0.1\n", + " generic = ['i cannot', \"i don't\", 'no vulnerabilities found', 'the code looks safe']\n", + " if any(p in text.lower() for p in generic): reward -= 0.5\n", + " rewards.append(max(-1.0, min(1.0, reward)))\n", + " return rewards\n", + "\n", + "print('āœ… 4 reward functions defined: format, severity, category, quality')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Cell 7: Initialize GRPO Trainer\n", + "from trl import GRPOTrainer, GRPOConfig\n", + "\n", + "config = GRPOConfig(\n", + " output_dir=OUTPUT_DIR,\n", + " num_train_epochs=NUM_EPOCHS,\n", + " per_device_train_batch_size=BATCH_SIZE,\n", + " gradient_accumulation_steps=GRAD_ACCUM,\n", + " num_generations=NUM_GENERATIONS,\n", + " max_completion_length=MAX_COMPLETION_LENGTH,\n", + " learning_rate=LEARNING_RATE,\n", + " beta=BETA,\n", + " scale_rewards=True,\n", + " reward_weights=[0.25, 0.25, 0.25, 0.25],\n", + " gradient_checkpointing=True,\n", + " bf16=True,\n", + " logging_steps=10,\n", + " logging_first_step=True,\n", + " logging_strategy='steps',\n", + " disable_tqdm=False, # Show progress bar in Colab\n", + " save_strategy='steps',\n", + " save_steps=SAVE_STEPS,\n", + " save_total_limit=2,\n", + " push_to_hub=False, # We push manually at the end\n", + " log_completions=False,\n", + " report_to='none',\n", + " seed=42,\n", + ")\n", + "\n", + "print('Initializing GRPOTrainer...')\n", + "trainer = GRPOTrainer(\n", + " model=MODEL_NAME,\n", + " args=config,\n", + " reward_funcs=[format_reward, severity_reward, category_reward, quality_reward],\n", + " train_dataset=train_dataset,\n", + ")\n", + "print(f'āœ… GRPOTrainer ready! {len(train_dataset)} samples, ~{len(train_dataset) // (BATCH_SIZE * GRAD_ACCUM)} steps')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Cell 8: TRAIN! šŸš€\n", + "# This takes 4-6 hours on T4. Colab will keep running if you stay connected.\n", + "# Tip: Keep the tab open and active to prevent disconnection.\n", + "\n", + "import time\n", + "start = time.time()\n", + "print('šŸš€ Starting GRPO V2 training...')\n", + "print(f'Estimated time: ~{len(train_dataset) / (BATCH_SIZE * GRAD_ACCUM) * 45 / 3600:.1f} hours')\n", + "print()\n", + "\n", + "trainer.train()\n", + "\n", + "elapsed = time.time() - start\n", + "print(f'\\nāœ… Training complete in {elapsed/3600:.1f} hours!')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Cell 9: Save and push to Hub\n", + "import os\n", + "from huggingface_hub import HfApi\n", + "\n", + "print(f'Saving model to {OUTPUT_DIR}...')\n", + "trainer.save_model(OUTPUT_DIR)\n", + "\n", + "print(f'Pushing to Hub: {HUB_MODEL_ID}...')\n", + "api = HfApi()\n", + "api.create_repo(repo_id=HUB_MODEL_ID, exist_ok=True)\n", + "\n", + "# Upload model files (skip checkpoints and optimizer states to save time)\n", + "api.upload_folder(\n", + " folder_path=OUTPUT_DIR,\n", + " repo_id=HUB_MODEL_ID,\n", + " commit_message='GRPO V2 ā€” trained on real audit findings, 4 reward functions',\n", + " ignore_patterns=['checkpoint-*', '*.pt'], # Skip checkpoints\n", + ")\n", + "\n", + "print(f'\\nšŸŽ‰ Model pushed to https://huggingface.co/{HUB_MODEL_ID}')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Cell 10: Quick inference test\n", + "from transformers import pipeline as hf_pipeline\n", + "\n", + "print('Loading trained model for inference...')\n", + "pipe = hf_pipeline('text-generation', model=OUTPUT_DIR, device=0, torch_dtype=torch.bfloat16)\n", + "\n", + "test_contract = \"\"\"\n", + "pragma solidity ^0.8.0;\n", + "\n", + "contract SimpleBank {\n", + " mapping(address => uint256) public balances;\n", + "\n", + " function deposit() public payable {\n", + " balances[msg.sender] += msg.value;\n", + " }\n", + "\n", + " function withdraw(uint256 amount) public {\n", + " require(balances[msg.sender] >= amount);\n", + " (bool success, ) = msg.sender.call{value: amount}(\\\"\\\");\n", + " require(success);\n", + " balances[msg.sender] -= amount;\n", + " }\n", + "}\n", + "\"\"\"\n", + "\n", + "messages = [\n", + " {'role': 'system', 'content': 'You are an expert smart contract security auditor. Analyze the provided Solidity code for vulnerabilities.'},\n", + " {'role': 'user', 'content': f'Audit this contract:\\n```solidity\\n{test_contract}\\n```'},\n", + "]\n", + "\n", + "result = pipe(messages, max_new_tokens=512, do_sample=False, return_full_text=False)\n", + "output = result[0]['generated_text']\n", + "if isinstance(output, list):\n", + " output = output[-1]['content']\n", + "\n", + "print('\\n=== Audit Result ===')\n", + "print(output)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "---\n", + "\n", + "## šŸŽ‰ Done!\n", + "\n", + "Your V2 model is now pushed to the Hub. Test it interactively at:\n", + "\n", + "**Demo Space:** [oxdev/security-auditor-demo](https://huggingface.co/spaces/oxdev/security-auditor-demo)\n", + "\n", + "**Model:** [oxdev/security-auditor-grpo](https://huggingface.co/oxdev/security-auditor-grpo)\n", + "\n", + "### Next Steps\n", + "- Train on more data: increase `SUBSET_SIZE` to 5000 or 10000\n", + "- Use a bigger model: try `Qwen/Qwen2.5-Coder-1.5B-Instruct` (needs A100)\n", + "- Fine-tune rewards: adjust weights in `reward_weights`\n", + "- Try different hyperparameters: learning rate, beta, num_generations" + ] + } + ] +} diff --git a/training_args.bin b/training_args.bin new file mode 100644 index 0000000..93d9ca8 --- /dev/null +++ b/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b51f3856815802830b7add9e23ddd089207e5c9941078dd606f120af0f983d09 +size 6776