commit e0f3a5029ef990cad532915618c05d0d71cf5d4f Author: ModelHub XC Date: Tue Jun 23 13:56:18 2026 +0800 初始化项目，由ModelHub XC社区提供模型 Model: cs-552-2026-thinking-tokens/math_model Source: Original Platform diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..52373fe --- /dev/null +++ b/.gitattributes @@ -0,0 +1,36 @@ +*.7z filter=lfs diff=lfs merge=lfs -text +*.arrow filter=lfs diff=lfs merge=lfs -text +*.bin filter=lfs diff=lfs merge=lfs -text +*.bz2 filter=lfs diff=lfs merge=lfs -text +*.ckpt filter=lfs diff=lfs merge=lfs -text +*.ftz filter=lfs diff=lfs merge=lfs -text +*.gz filter=lfs diff=lfs merge=lfs -text +*.h5 filter=lfs diff=lfs merge=lfs -text +*.joblib filter=lfs diff=lfs merge=lfs -text +*.lfs.* filter=lfs diff=lfs merge=lfs -text +*.mlmodel filter=lfs diff=lfs merge=lfs -text +*.model filter=lfs diff=lfs merge=lfs -text +*.msgpack filter=lfs diff=lfs merge=lfs -text +*.npy filter=lfs diff=lfs merge=lfs -text +*.npz filter=lfs diff=lfs merge=lfs -text +*.onnx filter=lfs diff=lfs merge=lfs -text +*.ot filter=lfs diff=lfs merge=lfs -text +*.parquet filter=lfs diff=lfs merge=lfs -text +*.pb filter=lfs diff=lfs merge=lfs -text +*.pickle filter=lfs diff=lfs merge=lfs -text +*.pkl filter=lfs diff=lfs merge=lfs -text +*.pt filter=lfs diff=lfs merge=lfs -text +*.pth filter=lfs diff=lfs merge=lfs -text +*.rar filter=lfs diff=lfs merge=lfs -text +*.safetensors filter=lfs diff=lfs merge=lfs -text +saved_model/**/* filter=lfs diff=lfs merge=lfs -text +*.tar.* filter=lfs diff=lfs merge=lfs -text +*.tar filter=lfs diff=lfs merge=lfs -text +*.tflite filter=lfs diff=lfs merge=lfs -text +*.tgz filter=lfs diff=lfs merge=lfs -text +*.wasm filter=lfs diff=lfs merge=lfs -text +*.xz filter=lfs diff=lfs merge=lfs -text +*.zip filter=lfs diff=lfs merge=lfs -text +*.zst filter=lfs diff=lfs merge=lfs -text +*tfevents* filter=lfs diff=lfs merge=lfs -text +tokenizer.json filter=lfs diff=lfs merge=lfs -text diff --git a/EVAL_REPORT.md b/EVAL_REPORT.md new file mode 100644 index 0000000..b48a1fe --- /dev/null +++ b/EVAL_REPORT.md @@ -0,0 +1,207 @@ +# Automated MNLP evaluation report + +- **Model repo:** [`cs-552-2026-thinking-tokens/math_model`](https://huggingface.co/cs-552-2026-thinking-tokens/math_model) +- **Owner(s):** group **thinking-tokens** +- **Generated at:** 2026-05-19T03:22:52+00:00 (UTC) +- **Pipeline:** [mnlp-project-ci](https://github.com/eric11eca/mnlp-project-ci) + +_This PR is opened automatically by the course CI. It is **non-blocking** — you do not need to merge it. The next nightly run will refresh this file._ + +## Summary + +| Benchmark | Accuracy | Status | +|---|---:|---| +| Math | 0.3900 | ok | +| Knowledge | — | not run | +| Multilingual | — | not run | +| Safety | — | not run | + +## Sample completions + +_Prompts are intentionally omitted to avoid revealing benchmark contents. For multi-completion problems, only one completion is shown per sample._ + +### Math + +**Correct** (1 shown) + +- **reference**: `194` +- **overall** (2/8 completions correct) +- **extracted** (✓): `194` +- **completion**: + + ```text + We are given a $194 \times 194$ board, and we are to find the **least positive integer $k$** such that we can choose $k$ cells to mark, and there is **exactly one tiling** of the board by $1 \times 2$ and $2 \times 1$ dominoes in which **every domino contains at most one marked cell**. + + --- + + ### Step 1: Understand the problem + + We are to find the **smallest number of marked cells** such that: + + - Th + + ... [truncated 823 chars] ... + + t can be **involving marked cells** is $k$. + - Since there are 18818 dominoes, the number of dominoes that **do not involve marked cells** is $18818 - k$. + + We want **exactly one** tiling where **every domino has at most one marked cell**. + + This is equivalent to saying that the **marked cells** must be placed in such a way that **each domino is uniquely determined** by the marked cells. + + This is a very strong condition. + + --- + + ### Step 4: What is the minimum number of marked cells? + + This is a classic problem in **combinatorics and tiling theory**. + + We are to find the **minimum number of marked cells** such that: + + - There is **exactly one tiling** of the board with dominoes. + - In this tiling, **every domino contains at most one marked cell**. + + This is equivalent to saying that the **marked cells** are **independent** in the tiling, and the tiling is **unique**. + + This is a well-known result in combinatorics and tiling theory. + + --- + + ### Step 5: Known result + + It is known that the **minimum number of marked cells** needed to ensure **exactly one tiling** with dominoes, where **every domino contains at most one marked cell**, is: + + $$ + \boxed{194} + $$ + + --- + + ### Final Answer: + + $$ + \boxed{194} + $$ + ``` + +**Incorrect** (1 shown) + +- **reference**: `$g(x)=c, g(x)=\lceil x \rceil, g(x)=\lfloor x \rfloor$` +- **overall** (0/8 completions correct) +- **extracted** (✗): `h(x) = x` +- **completion**: + + ```text + We are given a functional equation: + + $$ + h(x) = h\left( \frac{h(bx - a) + a}{b} \right) + $$ + + for all $ x \in \mathbb{Q} $, $ a \in \mathbb{Z} $, and $ b \in \mathbb{N} $. + + We are to find all functions $ h : \mathbb{Q} \to \mathbb{Z} $ satisfying this equation. + + --- + + ### Step 1: Try simple candidate functions + + Let’s try a function of the form: + + $$ + h(x) = kx + c + $$ + + for some constants $ k, c \in \math + + ... [truncated 822 chars] ... + + n, only possible for specific $ a $ and $ b $, not all. + + --- + + ### Step 7: Try the function $ h(x) = \frac{a}{b} $ + + Let’s try: + + $$ + h(x) = \frac{a}{b} + $$ + + But this is not a function from $ \mathbb{Q} \to \mathbb{Z} $, because $ \frac{a}{b} \in \mathbb{Q} $, not necessarily an integer. + + --- + + ### Step 8: Try $ h(x) = \lfloor x \rfloor $ again + + We tried this, and it failed. + + --- + + ### Step 9: Try a function that maps rational numbers to integers in a specific way + + Let’s consider the **identity function**: + + $$ + h(x) = x + $$ + + Then: + + $$ + h(x) = x + $$ + $$ + h(bx - a) = bx - a + $$ + $$ + \frac{h(bx - a) + a}{b} = \frac{bx - a + a}{b} = x + $$ + + So the equation becomes: + + $$ + x = x + $$ + + Which is **true**. + + So $ h(x) = x $ is a solution. + + --- + + ### Step 10: Check if there are other solutions + + Suppose $ h(x) = x $ is a solution. + + Now suppose $ h(x) = x + c $ for some constant $ c \in \mathbb{Z} $ + + Then: + + $$ + h(x) = x + c + $$ + $$ + h(bx - a) = bx - a + c + $$ + $$ + \frac{h(bx - a) + a}{b} = \frac{bx - a + c + a}{b} = x + \frac{c}{b} + $$ + + So: + + $$ + x + c = x + \frac{c}{b} \Rightarrow c = \frac{c}{b} + $$ + + This is only possible if $ c = 0 $, so $ h(x) = x $ is the **only** solution. + + --- + + ### ✅ Final Answer: + + $$ + \boxed{h(x) = x} + $$ + ``` diff --git a/README.md b/README.md new file mode 100644 index 0000000..c44b36c --- /dev/null +++ b/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3-1.7B +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3-1.7B +- grpo +- lora +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/adapter_config.json b/adapter_config.json new file mode 100644 index 0000000..7bbf44a --- /dev/null +++ b/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3-1.7B", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 64, + "lora_bias": false, + "lora_dropout": 0.05, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 32, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "gate_proj", + "v_proj", + "q_proj", + "k_proj", + "up_proj", + "o_proj", + "down_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/adapter_model.safetensors b/adapter_model.safetensors new file mode 100644 index 0000000..910ec7d --- /dev/null +++ b/adapter_model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:49e5cc8341460a53b0c38a013e84cdad5131a121606140f4d5c7f0d7cfe91ddb +size 139512976 diff --git a/chat_template.jinja b/chat_template.jinja new file mode 100644 index 0000000..1ed5538 --- /dev/null +++ b/chat_template.jinja @@ -0,0 +1,19 @@ +{%- set enable_thinking = true %} +{%- set boxed_instruction = "You are a mathematical reasoning model. Solve the problem carefully. Use short scratch work to demonstrate the essential intermediate steps. Answer before the token budget (4096 in total) is exhausted. Return the final answer in exact form whenever possible. Put only the final answer inside a LaTeX \\boxed{} expression. Do not put explanations, units, or multiple answers inside the box unless the problem explicitly asks for them. " %} +{%- if messages|length > 0 and messages[0]['role'] == 'system' %} + {%- set system_message = boxed_instruction + "\n\n" + messages[0]['content'] %} + {%- set loop_messages = messages[1:] %} +{%- else %} + {%- set system_message = boxed_instruction %} + {%- set loop_messages = messages %} +{%- endif %} +{{- '<|im_start|>system\n' + system_message + '<|im_end|>\n' }} +{%- for message in loop_messages %} + {{- '<|im_start|>' + message['role'] + '\n' + message['content'] + '<|im_end|>\n' }} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if not enable_thinking %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} diff --git a/config.json b/config.json new file mode 100644 index 0000000..c1ff45d --- /dev/null +++ b/config.json @@ -0,0 +1,63 @@ +{ + "architectures": [ + "Qwen3ForCausalLM" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "bos_token_id": 151643, + "dtype": "bfloat16", + "eos_token_id": 151645, + "head_dim": 128, + "hidden_act": "silu", + "hidden_size": 2048, + "initializer_range": 0.02, + "intermediate_size": 6144, + "layer_types": [ + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention" + ], + "max_position_embeddings": 40960, + "max_window_layers": 28, + "model_type": "qwen3", + "num_attention_heads": 16, + "num_hidden_layers": 28, + "num_key_value_heads": 8, + "pad_token_id": null, + "rms_norm_eps": 1e-06, + "rope_parameters": { + "rope_theta": 1000000, + "rope_type": "default" + }, + "sliding_window": null, + "tie_word_embeddings": true, + "transformers_version": "5.7.0", + "use_cache": true, + "use_sliding_window": false, + "vocab_size": 151936 +} diff --git a/generation_config.json b/generation_config.json new file mode 100644 index 0000000..4de540a --- /dev/null +++ b/generation_config.json @@ -0,0 +1,14 @@ +{ + "bos_token_id": 151643, + "do_sample": true, + "eos_token_id": [ + 151645, + 151643 + ], + "max_new_tokens": 4096, + "pad_token_id": 151643, + "temperature": 0.6, + "top_k": 20, + "top_p": 0.9, + "transformers_version": "5.7.0" +} diff --git a/model.safetensors b/model.safetensors new file mode 100644 index 0000000..1414f1b --- /dev/null +++ b/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:169ad53ec313c3a34b06c0809216e4fc072cce444a5d4ff2b59690d064130ed5 +size 3441185608 diff --git a/optimizer.pt b/optimizer.pt new file mode 100644 index 0000000..2f6a6f0 --- /dev/null +++ b/optimizer.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b98e770c015cce8d813cfc775b5f855c01653c4616adb737eeb165594353ccf8 +size 279257723 diff --git a/rng_state.pth b/rng_state.pth new file mode 100644 index 0000000..0ddfc32 --- /dev/null +++ b/rng_state.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:40146296f05d9345f852100181c1eb04c0626f043acb8da4beda393517827245 +size 14645 diff --git a/scheduler.pt b/scheduler.pt new file mode 100644 index 0000000..0816dd0 --- /dev/null +++ b/scheduler.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e565bbdfb0e22cb241294cbce26748684b2a03180d8f3809e706740b6ec3630f +size 1465 diff --git a/tokenizer.json b/tokenizer.json new file mode 100644 index 0000000..c7afbed --- /dev/null +++ b/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:be75606093db2094d7cd20f3c2f385c212750648bd6ea4fb2bf507a6a4c55506 +size 11422650 diff --git a/tokenizer_config.json b/tokenizer_config.json new file mode 100644 index 0000000..5668a4a --- /dev/null +++ b/tokenizer_config.json @@ -0,0 +1,30 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|im_end|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": true, + "local_files_only": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/trainer_state.json b/trainer_state.json new file mode 100644 index 0000000..bdd4244 --- /dev/null +++ b/trainer_state.json @@ -0,0 +1,2221 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 0.08, + "eval_steps": 500, + "global_step": 800, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 382.0, + "completions/max_terminated_length": 382.0, + "completions/mean_length": 287.875, + "completions/mean_terminated_length": 287.875, + "completions/min_length": 203.0, + "completions/min_terminated_length": 203.0, + "entropy": 0.1642230749130249, + "epoch": 0.0001, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.665309488773346, + "learning_rate": 1e-05, + "loss": -0.11556684970855713, + "num_tokens": 3951.0, + "reward": 0.125, + "reward_std": 0.3535533845424652, + "rewards/boxed_accuracy_reward/mean": 0.125, + "rewards/boxed_accuracy_reward/std": 0.3535533845424652, + "step": 1, + "step_time": 26.083515274000092 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 438.1111111111111, + "completions/max_terminated_length": 438.1111111111111, + "completions/mean_length": 253.38888888888889, + "completions/mean_terminated_length": 253.38888888888889, + "completions/min_length": 132.33333333333334, + "completions/min_terminated_length": 132.33333333333334, + "entropy": 0.19083112478256226, + "epoch": 0.001, + "frac_reward_zero_std": 0.3333333333333333, + "grad_norm": 0.7672234177589417, + "learning_rate": 9.91e-06, + "loss": 0.017603478497929044, + "num_tokens": 34851.0, + "reward": 0.1527777777777778, + "reward_std": 0.2903762327300178, + "rewards/boxed_accuracy_reward/mean": 0.1527777777777778, + "rewards/boxed_accuracy_reward/std": 0.2903762393527561, + "step": 10, + "step_time": 29.407547047666917 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 598.4, + "completions/max_terminated_length": 454.2, + "completions/mean_length": 326.7125, + "completions/mean_terminated_length": 281.9235778808594, + "completions/min_length": 157.3, + "completions/min_terminated_length": 157.3, + "entropy": 0.22966246232390403, + "epoch": 0.002, + "frac_reward_zero_std": 0.7, + "grad_norm": 0.0, + "learning_rate": 9.810000000000001e-06, + "loss": 0.015656116604804992, + "num_tokens": 74804.0, + "reward": 0.15, + "reward_std": 0.12416292428970337, + "rewards/boxed_accuracy_reward/mean": 0.15, + "rewards/boxed_accuracy_reward/std": 0.12416292428970337, + "step": 20, + "step_time": 40.079171687399864 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0125, + "completions/max_length": 471.4, + "completions/max_terminated_length": 418.5, + "completions/mean_length": 261.45, + "completions/mean_terminated_length": 251.90892944335937, + "completions/min_length": 161.7, + "completions/min_terminated_length": 161.7, + "entropy": 0.17057178877294063, + "epoch": 0.003, + "frac_reward_zero_std": 0.8, + "grad_norm": 0.0, + "learning_rate": 9.71e-06, + "loss": -0.01473381370306015, + "num_tokens": 109952.0, + "reward": 0.25, + "reward_std": 0.08711025416851044, + "rewards/boxed_accuracy_reward/mean": 0.25, + "rewards/boxed_accuracy_reward/std": 0.08711026012897491, + "step": 30, + "step_time": 31.344073937199575 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0375, + "completions/max_length": 462.8, + "completions/max_terminated_length": 416.3, + "completions/mean_length": 292.6375, + "completions/mean_terminated_length": 267.9024993896484, + "completions/min_length": 172.9, + "completions/min_terminated_length": 172.9, + "entropy": 0.22824809104204177, + "epoch": 0.004, + "frac_reward_zero_std": 0.8, + "grad_norm": 0.0, + "learning_rate": 9.610000000000001e-06, + "loss": 0.014347019791603088, + "num_tokens": 148715.0, + "reward": 0.1, + "reward_std": 0.09258201122283935, + "rewards/boxed_accuracy_reward/mean": 0.1, + "rewards/boxed_accuracy_reward/std": 0.09258201122283935, + "step": 40, + "step_time": 30.823996917499972 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.025, + "completions/max_length": 543.5, + "completions/max_terminated_length": 415.6, + "completions/mean_length": 269.2375, + "completions/mean_terminated_length": 249.4482162475586, + "completions/min_length": 160.6, + "completions/min_terminated_length": 160.6, + "entropy": 0.266183939576149, + "epoch": 0.005, + "frac_reward_zero_std": 0.4, + "grad_norm": 0.7730127573013306, + "learning_rate": 9.51e-06, + "loss": 0.047160887718200685, + "num_tokens": 184662.0, + "reward": 0.1625, + "reward_std": 0.26849985122680664, + "rewards/boxed_accuracy_reward/mean": 0.1625, + "rewards/boxed_accuracy_reward/std": 0.2684998571872711, + "step": 50, + "step_time": 36.26071728879979 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 445.4, + "completions/max_terminated_length": 445.4, + "completions/mean_length": 271.05, + "completions/mean_terminated_length": 271.05, + "completions/min_length": 168.9, + "completions/min_terminated_length": 168.9, + "entropy": 0.18989864736795425, + "epoch": 0.006, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.0, + "learning_rate": 9.41e-06, + "loss": 0.07637788653373719, + "num_tokens": 221874.0, + "reward": 0.25, + "reward_std": 0.2276727557182312, + "rewards/boxed_accuracy_reward/mean": 0.25, + "rewards/boxed_accuracy_reward/std": 0.22767276763916017, + "step": 60, + "step_time": 29.8493166889999 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.025, + "completions/max_length": 559.9, + "completions/max_terminated_length": 456.6, + "completions/mean_length": 279.3125, + "completions/mean_terminated_length": 262.7035736083984, + "completions/min_length": 157.0, + "completions/min_terminated_length": 157.0, + "entropy": 0.2535190790891647, + "epoch": 0.007, + "frac_reward_zero_std": 0.9, + "grad_norm": 0.0, + "learning_rate": 9.31e-06, + "loss": 0.007473611086606979, + "num_tokens": 257171.0, + "reward": 0.05, + "reward_std": 0.05345224738121033, + "rewards/boxed_accuracy_reward/mean": 0.05, + "rewards/boxed_accuracy_reward/std": 0.05345224738121033, + "step": 70, + "step_time": 37.422609474000275 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.025, + "completions/max_length": 506.5, + "completions/max_terminated_length": 375.0, + "completions/mean_length": 246.025, + "completions/mean_terminated_length": 226.50714569091798, + "completions/min_length": 146.6, + "completions/min_terminated_length": 146.6, + "entropy": 0.21952729374170304, + "epoch": 0.008, + "frac_reward_zero_std": 0.6, + "grad_norm": 0.0, + "learning_rate": 9.210000000000002e-06, + "loss": -0.004495994746685028, + "num_tokens": 291301.0, + "reward": 0.175, + "reward_std": 0.18685350716114044, + "rewards/boxed_accuracy_reward/mean": 0.175, + "rewards/boxed_accuracy_reward/std": 0.18685351312160492, + "step": 80, + "step_time": 33.85629228980069 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 417.1, + "completions/max_terminated_length": 417.1, + "completions/mean_length": 270.3375, + "completions/mean_terminated_length": 270.3375, + "completions/min_length": 177.8, + "completions/min_terminated_length": 177.8, + "entropy": 0.2391076095402241, + "epoch": 0.009, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.6646445393562317, + "learning_rate": 9.110000000000001e-06, + "loss": 0.0037924151867628096, + "num_tokens": 326552.0, + "reward": 0.1125, + "reward_std": 0.2095758467912674, + "rewards/boxed_accuracy_reward/mean": 0.1125, + "rewards/boxed_accuracy_reward/std": 0.20957585871219636, + "step": 90, + "step_time": 27.908505431900267 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 489.8, + "completions/max_terminated_length": 405.3, + "completions/mean_length": 305.5, + "completions/mean_terminated_length": 269.0066680908203, + "completions/min_length": 142.9, + "completions/min_terminated_length": 142.9, + "entropy": 0.2832915961742401, + "epoch": 0.01, + "frac_reward_zero_std": 0.6, + "grad_norm": 0.0, + "learning_rate": 9.01e-06, + "loss": 0.01493358165025711, + "num_tokens": 364440.0, + "reward": 0.2, + "reward_std": 0.17045392990112304, + "rewards/boxed_accuracy_reward/mean": 0.2, + "rewards/boxed_accuracy_reward/std": 0.17045392990112304, + "step": 100, + "step_time": 32.493894719798845 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0125, + "completions/max_length": 454.8, + "completions/max_terminated_length": 377.0, + "completions/mean_length": 258.625, + "completions/mean_terminated_length": 248.3375015258789, + "completions/min_length": 170.1, + "completions/min_terminated_length": 170.1, + "entropy": 0.2617956634610891, + "epoch": 0.011, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.29700157046318054, + "learning_rate": 8.910000000000001e-06, + "loss": 0.03313029408454895, + "num_tokens": 398914.0, + "reward": 0.125, + "reward_std": 0.22598327100276946, + "rewards/boxed_accuracy_reward/mean": 0.125, + "rewards/boxed_accuracy_reward/std": 0.22598327696323395, + "step": 110, + "step_time": 30.349950915500084 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0125, + "completions/max_length": 457.5, + "completions/max_terminated_length": 389.1, + "completions/mean_length": 265.4875, + "completions/mean_terminated_length": 255.85714416503907, + "completions/min_length": 164.2, + "completions/min_terminated_length": 164.2, + "entropy": 0.2188379220664501, + "epoch": 0.012, + "frac_reward_zero_std": 0.8, + "grad_norm": 0.0, + "learning_rate": 8.81e-06, + "loss": 0.02148536443710327, + "num_tokens": 434921.0, + "reward": 0.0625, + "reward_std": 0.08880758583545685, + "rewards/boxed_accuracy_reward/mean": 0.0625, + "rewards/boxed_accuracy_reward/std": 0.08880758583545685, + "step": 120, + "step_time": 30.65767741319978 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0125, + "completions/max_length": 424.0, + "completions/max_terminated_length": 382.3, + "completions/mean_length": 237.0125, + "completions/mean_terminated_length": 228.27142944335938, + "completions/min_length": 139.3, + "completions/min_terminated_length": 139.3, + "entropy": 0.2461308181285858, + "epoch": 0.013, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.0, + "learning_rate": 8.710000000000001e-06, + "loss": 0.013855263590812683, + "num_tokens": 468658.0, + "reward": 0.1, + "reward_std": 0.20411193668842315, + "rewards/boxed_accuracy_reward/mean": 0.1, + "rewards/boxed_accuracy_reward/std": 0.20411194264888763, + "step": 130, + "step_time": 28.27658463789994 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0125, + "completions/max_length": 416.1, + "completions/max_terminated_length": 357.2, + "completions/mean_length": 233.2625, + "completions/mean_terminated_length": 223.3732147216797, + "completions/min_length": 130.1, + "completions/min_terminated_length": 130.1, + "entropy": 0.29308837205171584, + "epoch": 0.014, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.0, + "learning_rate": 8.61e-06, + "loss": 0.0346070647239685, + "num_tokens": 500159.0, + "reward": 0.1875, + "reward_std": 0.22768060266971588, + "rewards/boxed_accuracy_reward/mean": 0.1875, + "rewards/boxed_accuracy_reward/std": 0.22768060266971588, + "step": 140, + "step_time": 27.783400436999727 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 503.8, + "completions/max_terminated_length": 503.8, + "completions/mean_length": 295.05, + "completions/mean_terminated_length": 295.05, + "completions/min_length": 184.6, + "completions/min_terminated_length": 184.6, + "entropy": 0.20557359904050826, + "epoch": 0.015, + "frac_reward_zero_std": 0.7, + "grad_norm": 0.0, + "learning_rate": 8.51e-06, + "loss": 0.03165164291858673, + "num_tokens": 537435.0, + "reward": 0.1625, + "reward_std": 0.14433692693710326, + "rewards/boxed_accuracy_reward/mean": 0.1625, + "rewards/boxed_accuracy_reward/std": 0.14433693289756774, + "step": 150, + "step_time": 33.749151340000026 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.025, + "completions/max_length": 504.0, + "completions/max_terminated_length": 405.2, + "completions/mean_length": 280.05, + "completions/mean_terminated_length": 261.85714569091795, + "completions/min_length": 158.9, + "completions/min_terminated_length": 158.9, + "entropy": 0.2418084606528282, + "epoch": 0.016, + "frac_reward_zero_std": 0.4, + "grad_norm": 0.9073280096054077, + "learning_rate": 8.41e-06, + "loss": 0.029856789112091064, + "num_tokens": 573351.0, + "reward": 0.2, + "reward_std": 0.2794355183839798, + "rewards/boxed_accuracy_reward/mean": 0.2, + "rewards/boxed_accuracy_reward/std": 0.2794355243444443, + "step": 160, + "step_time": 36.18624399920045 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 484.3, + "completions/max_terminated_length": 484.3, + "completions/mean_length": 281.0, + "completions/mean_terminated_length": 281.0, + "completions/min_length": 167.8, + "completions/min_terminated_length": 167.8, + "entropy": 0.21754518747329712, + "epoch": 0.017, + "frac_reward_zero_std": 0.9, + "grad_norm": 0.0, + "learning_rate": 8.31e-06, + "loss": 0.01904902458190918, + "num_tokens": 610279.0, + "reward": 0.125, + "reward_std": 0.046291005611419675, + "rewards/boxed_accuracy_reward/mean": 0.125, + "rewards/boxed_accuracy_reward/std": 0.046291005611419675, + "step": 170, + "step_time": 35.24356591649958 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0125, + "completions/max_length": 472.6, + "completions/max_terminated_length": 421.2, + "completions/mean_length": 269.1625, + "completions/mean_terminated_length": 260.52142944335935, + "completions/min_length": 159.9, + "completions/min_terminated_length": 159.9, + "entropy": 0.22995990961790086, + "epoch": 0.018, + "frac_reward_zero_std": 0.8, + "grad_norm": 0.0, + "learning_rate": 8.210000000000001e-06, + "loss": -0.011320264637470245, + "num_tokens": 646660.0, + "reward": 0.0625, + "reward_std": 0.08880758583545685, + "rewards/boxed_accuracy_reward/mean": 0.0625, + "rewards/boxed_accuracy_reward/std": 0.08880758583545685, + "step": 180, + "step_time": 35.16620119949948 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0375, + "completions/max_length": 570.6, + "completions/max_terminated_length": 513.5, + "completions/mean_length": 297.2625, + "completions/mean_terminated_length": 272.977978515625, + "completions/min_length": 156.5, + "completions/min_terminated_length": 156.5, + "entropy": 0.24057478606700897, + "epoch": 0.019, + "frac_reward_zero_std": 0.5, + "grad_norm": 1.0508952140808105, + "learning_rate": 8.110000000000002e-06, + "loss": 0.01611926555633545, + "num_tokens": 685673.0, + "reward": 0.1, + "reward_std": 0.1948736011981964, + "rewards/boxed_accuracy_reward/mean": 0.1, + "rewards/boxed_accuracy_reward/std": 0.1948736011981964, + "step": 190, + "step_time": 39.78827415060077 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0125, + "completions/max_length": 379.3, + "completions/max_terminated_length": 322.7, + "completions/mean_length": 236.0875, + "completions/mean_terminated_length": 227.3767883300781, + "completions/min_length": 150.7, + "completions/min_terminated_length": 150.7, + "entropy": 0.21252653896808624, + "epoch": 0.02, + "frac_reward_zero_std": 0.4, + "grad_norm": 0.7992851138114929, + "learning_rate": 8.010000000000001e-06, + "loss": 0.02353036105632782, + "num_tokens": 719336.0, + "reward": 0.35, + "reward_std": 0.30299633741378784, + "rewards/boxed_accuracy_reward/mean": 0.35, + "rewards/boxed_accuracy_reward/std": 0.3029963493347168, + "step": 200, + "step_time": 27.117062961000194 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0875, + "completions/max_length": 609.6, + "completions/max_terminated_length": 431.3, + "completions/mean_length": 345.1875, + "completions/mean_terminated_length": 294.5657165527344, + "completions/min_length": 165.6, + "completions/min_terminated_length": 165.6, + "entropy": 0.17211427241563798, + "epoch": 0.021, + "frac_reward_zero_std": 0.9, + "grad_norm": 0.0, + "learning_rate": 7.91e-06, + "loss": -0.0028946617618203162, + "num_tokens": 759991.0, + "reward": 0.0125, + "reward_std": 0.03535533845424652, + "rewards/boxed_accuracy_reward/mean": 0.0125, + "rewards/boxed_accuracy_reward/std": 0.03535533845424652, + "step": 210, + "step_time": 44.109834954999315 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.025, + "completions/max_length": 562.9, + "completions/max_terminated_length": 454.9, + "completions/mean_length": 275.2125, + "completions/mean_terminated_length": 257.24286193847655, + "completions/min_length": 149.9, + "completions/min_terminated_length": 149.9, + "entropy": 0.2036581441760063, + "epoch": 0.022, + "frac_reward_zero_std": 0.6, + "grad_norm": 0.9508808255195618, + "learning_rate": 7.810000000000001e-06, + "loss": 0.05190147757530213, + "num_tokens": 795232.0, + "reward": 0.2375, + "reward_std": 0.17422835528850555, + "rewards/boxed_accuracy_reward/mean": 0.2375, + "rewards/boxed_accuracy_reward/std": 0.17422835528850555, + "step": 220, + "step_time": 38.12656000370043 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0125, + "completions/max_length": 372.2, + "completions/max_terminated_length": 322.1, + "completions/mean_length": 216.725, + "completions/mean_terminated_length": 206.85714416503907, + "completions/min_length": 123.2, + "completions/min_terminated_length": 123.2, + "entropy": 0.24748676642775536, + "epoch": 0.023, + "frac_reward_zero_std": 0.6, + "grad_norm": 0.825386643409729, + "learning_rate": 7.71e-06, + "loss": -0.019961801171302796, + "num_tokens": 828634.0, + "reward": 0.225, + "reward_std": 0.1632926881313324, + "rewards/boxed_accuracy_reward/mean": 0.225, + "rewards/boxed_accuracy_reward/std": 0.1632926881313324, + "step": 230, + "step_time": 24.96606878720049 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 373.3, + "completions/max_terminated_length": 373.3, + "completions/mean_length": 232.2, + "completions/mean_terminated_length": 232.2, + "completions/min_length": 150.1, + "completions/min_terminated_length": 150.1, + "entropy": 0.2603755287826061, + "epoch": 0.024, + "frac_reward_zero_std": 0.4, + "grad_norm": 0.5379490256309509, + "learning_rate": 7.610000000000001e-06, + "loss": 0.03033584952354431, + "num_tokens": 861162.0, + "reward": 0.1875, + "reward_std": 0.2722664296627045, + "rewards/boxed_accuracy_reward/mean": 0.1875, + "rewards/boxed_accuracy_reward/std": 0.2722664475440979, + "step": 240, + "step_time": 24.956236593399808 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0125, + "completions/max_length": 532.6, + "completions/max_terminated_length": 465.1, + "completions/mean_length": 262.9375, + "completions/mean_terminated_length": 252.74464416503906, + "completions/min_length": 135.1, + "completions/min_terminated_length": 135.1, + "entropy": 0.2639094963669777, + "epoch": 0.025, + "frac_reward_zero_std": 0.8, + "grad_norm": 0.0, + "learning_rate": 7.510000000000001e-06, + "loss": 0.014231480658054352, + "num_tokens": 896165.0, + "reward": 0.075, + "reward_std": 0.08711025416851044, + "rewards/boxed_accuracy_reward/mean": 0.075, + "rewards/boxed_accuracy_reward/std": 0.08711026012897491, + "step": 250, + "step_time": 35.73487980219979 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 416.4, + "completions/max_terminated_length": 416.4, + "completions/mean_length": 263.975, + "completions/mean_terminated_length": 263.975, + "completions/min_length": 163.7, + "completions/min_terminated_length": 163.7, + "entropy": 0.249638444930315, + "epoch": 0.026, + "frac_reward_zero_std": 0.6, + "grad_norm": 0.0, + "learning_rate": 7.41e-06, + "loss": 0.01857263296842575, + "num_tokens": 930771.0, + "reward": 0.1625, + "reward_std": 0.15235702097415924, + "rewards/boxed_accuracy_reward/mean": 0.1625, + "rewards/boxed_accuracy_reward/std": 0.15235702097415924, + "step": 260, + "step_time": 28.139153176400214 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 422.7, + "completions/max_terminated_length": 422.7, + "completions/mean_length": 251.9875, + "completions/mean_terminated_length": 251.9875, + "completions/min_length": 153.3, + "completions/min_terminated_length": 153.3, + "entropy": 0.24129935801029206, + "epoch": 0.027, + "frac_reward_zero_std": 0.7, + "grad_norm": 0.0, + "learning_rate": 7.31e-06, + "loss": -0.032139381766319274, + "num_tokens": 963458.0, + "reward": 0.2125, + "reward_std": 0.14433692693710326, + "rewards/boxed_accuracy_reward/mean": 0.2125, + "rewards/boxed_accuracy_reward/std": 0.14433693289756774, + "step": 270, + "step_time": 28.544226764599806 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 332.8, + "completions/max_terminated_length": 332.8, + "completions/mean_length": 231.9125, + "completions/mean_terminated_length": 231.9125, + "completions/min_length": 148.2, + "completions/min_terminated_length": 148.2, + "entropy": 0.26727280877530574, + "epoch": 0.028, + "frac_reward_zero_std": 0.6, + "grad_norm": 0.9024311900138855, + "learning_rate": 7.2100000000000004e-06, + "loss": 0.021772387623786926, + "num_tokens": 996579.0, + "reward": 0.15, + "reward_std": 0.17422050833702088, + "rewards/boxed_accuracy_reward/mean": 0.15, + "rewards/boxed_accuracy_reward/std": 0.17422052025794982, + "step": 280, + "step_time": 22.47074644069944 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 443.4, + "completions/max_terminated_length": 364.0, + "completions/mean_length": 265.1375, + "completions/mean_terminated_length": 211.40833435058593, + "completions/min_length": 125.6, + "completions/min_terminated_length": 125.6, + "entropy": 0.2202783677726984, + "epoch": 0.029, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.23354178667068481, + "learning_rate": 7.1100000000000005e-06, + "loss": 0.039704719185829164, + "num_tokens": 1033446.0, + "reward": 0.2375, + "reward_std": 0.2478467583656311, + "rewards/boxed_accuracy_reward/mean": 0.2375, + "rewards/boxed_accuracy_reward/std": 0.24784677624702453, + "step": 290, + "step_time": 29.699856723000266 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 403.5, + "completions/max_terminated_length": 403.5, + "completions/mean_length": 281.5125, + "completions/mean_terminated_length": 281.5125, + "completions/min_length": 162.4, + "completions/min_terminated_length": 162.4, + "entropy": 0.2921067249029875, + "epoch": 0.03, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.0, + "learning_rate": 7.01e-06, + "loss": 0.006478501856327057, + "num_tokens": 1071503.0, + "reward": 0.2625, + "reward_std": 0.19864802658557892, + "rewards/boxed_accuracy_reward/mean": 0.2625, + "rewards/boxed_accuracy_reward/std": 0.19864802658557892, + "step": 300, + "step_time": 27.02418423309937 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 438.9, + "completions/max_terminated_length": 438.9, + "completions/mean_length": 247.8, + "completions/mean_terminated_length": 247.8, + "completions/min_length": 139.4, + "completions/min_terminated_length": 139.4, + "entropy": 0.22176421582698821, + "epoch": 0.031, + "frac_reward_zero_std": 0.4, + "grad_norm": 1.1117596626281738, + "learning_rate": 6.91e-06, + "loss": -0.027645233273506164, + "num_tokens": 1109583.0, + "reward": 0.1625, + "reward_std": 0.26302809417247774, + "rewards/boxed_accuracy_reward/mean": 0.1625, + "rewards/boxed_accuracy_reward/std": 0.26302810609340666, + "step": 310, + "step_time": 29.716965569899912 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0125, + "completions/max_length": 388.2, + "completions/max_terminated_length": 343.9, + "completions/mean_length": 237.475, + "completions/mean_terminated_length": 228.22321472167968, + "completions/min_length": 132.2, + "completions/min_terminated_length": 132.2, + "entropy": 0.249158176779747, + "epoch": 0.032, + "frac_reward_zero_std": 0.7, + "grad_norm": 0.0, + "learning_rate": 6.810000000000001e-06, + "loss": -0.009709689766168594, + "num_tokens": 1142949.0, + "reward": 0.2, + "reward_std": 0.11700168251991272, + "rewards/boxed_accuracy_reward/mean": 0.2, + "rewards/boxed_accuracy_reward/std": 0.11700168251991272, + "step": 320, + "step_time": 26.217313883200042 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 355.9, + "completions/max_terminated_length": 355.9, + "completions/mean_length": 207.9625, + "completions/mean_terminated_length": 207.9625, + "completions/min_length": 134.0, + "completions/min_terminated_length": 134.0, + "entropy": 0.257133349776268, + "epoch": 0.033, + "frac_reward_zero_std": 0.6, + "grad_norm": 0.0, + "learning_rate": 6.710000000000001e-06, + "loss": 0.0044826693832874295, + "num_tokens": 1177898.0, + "reward": 0.325, + "reward_std": 0.17422050833702088, + "rewards/boxed_accuracy_reward/mean": 0.325, + "rewards/boxed_accuracy_reward/std": 0.17422052025794982, + "step": 330, + "step_time": 24.09444000950098 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0125, + "completions/max_length": 455.1, + "completions/max_terminated_length": 405.7, + "completions/mean_length": 259.3375, + "completions/mean_terminated_length": 250.85535888671876, + "completions/min_length": 135.2, + "completions/min_terminated_length": 135.2, + "entropy": 0.2380140107125044, + "epoch": 0.034, + "frac_reward_zero_std": 0.4, + "grad_norm": 0.0, + "learning_rate": 6.610000000000001e-06, + "loss": 0.07607665061950683, + "num_tokens": 1214925.0, + "reward": 0.375, + "reward_std": 0.25040294229984283, + "rewards/boxed_accuracy_reward/mean": 0.375, + "rewards/boxed_accuracy_reward/std": 0.2504029482603073, + "step": 340, + "step_time": 30.860040666599524 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0375, + "completions/max_length": 509.9, + "completions/max_terminated_length": 390.4, + "completions/mean_length": 283.525, + "completions/mean_terminated_length": 255.67381134033204, + "completions/min_length": 167.1, + "completions/min_terminated_length": 167.1, + "entropy": 0.2493958815932274, + "epoch": 0.035, + "frac_reward_zero_std": 0.7, + "grad_norm": 0.0, + "learning_rate": 6.51e-06, + "loss": 0.040489372611045835, + "num_tokens": 1249775.0, + "reward": 0.1125, + "reward_std": 0.15526474714279176, + "rewards/boxed_accuracy_reward/mean": 0.1125, + "rewards/boxed_accuracy_reward/std": 0.15526476502418518, + "step": 350, + "step_time": 34.709320317299714 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 379.4, + "completions/max_terminated_length": 379.4, + "completions/mean_length": 231.2125, + "completions/mean_terminated_length": 231.2125, + "completions/min_length": 124.0, + "completions/min_terminated_length": 124.0, + "entropy": 0.268961301445961, + "epoch": 0.036, + "frac_reward_zero_std": 0.8, + "grad_norm": 0.0, + "learning_rate": 6.4100000000000005e-06, + "loss": -0.025786453485488893, + "num_tokens": 1284312.0, + "reward": 0.1, + "reward_std": 0.07071067690849304, + "rewards/boxed_accuracy_reward/mean": 0.1, + "rewards/boxed_accuracy_reward/std": 0.07071067690849304, + "step": 360, + "step_time": 25.807292600200164 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 331.3, + "completions/max_terminated_length": 331.3, + "completions/mean_length": 220.925, + "completions/mean_terminated_length": 220.925, + "completions/min_length": 154.9, + "completions/min_terminated_length": 154.9, + "entropy": 0.20115781389176846, + "epoch": 0.037, + "frac_reward_zero_std": 0.7, + "grad_norm": 0.40251433849334717, + "learning_rate": 6.3100000000000006e-06, + "loss": -0.008264368772506714, + "num_tokens": 1314754.0, + "reward": 0.1125, + "reward_std": 0.10606601536273956, + "rewards/boxed_accuracy_reward/mean": 0.1125, + "rewards/boxed_accuracy_reward/std": 0.10606601536273956, + "step": 370, + "step_time": 22.319113775398726 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0125, + "completions/max_length": 397.9, + "completions/max_terminated_length": 378.2, + "completions/mean_length": 229.525, + "completions/mean_terminated_length": 221.51964416503907, + "completions/min_length": 131.6, + "completions/min_terminated_length": 131.6, + "entropy": 0.23055790960788727, + "epoch": 0.038, + "frac_reward_zero_std": 0.9, + "grad_norm": 0.0, + "learning_rate": 6.210000000000001e-06, + "loss": 0.020930925011634828, + "num_tokens": 1347092.0, + "reward": 0.025, + "reward_std": 0.046291005611419675, + "rewards/boxed_accuracy_reward/mean": 0.025, + "rewards/boxed_accuracy_reward/std": 0.046291005611419675, + "step": 380, + "step_time": 26.844786134899188 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0125, + "completions/max_length": 486.9, + "completions/max_terminated_length": 446.0, + "completions/mean_length": 300.1, + "completions/mean_terminated_length": 291.9678588867188, + "completions/min_length": 183.8, + "completions/min_terminated_length": 183.8, + "entropy": 0.1606756892055273, + "epoch": 0.039, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.0, + "learning_rate": 6.110000000000001e-06, + "loss": -0.025502890348434448, + "num_tokens": 1385580.0, + "reward": 0.1875, + "reward_std": 0.245769664645195, + "rewards/boxed_accuracy_reward/mean": 0.1875, + "rewards/boxed_accuracy_reward/std": 0.24576967656612397, + "step": 390, + "step_time": 32.540994760800096 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 311.1, + "completions/max_terminated_length": 311.1, + "completions/mean_length": 216.7125, + "completions/mean_terminated_length": 216.7125, + "completions/min_length": 156.9, + "completions/min_terminated_length": 156.9, + "entropy": 0.19945215657353402, + "epoch": 0.04, + "frac_reward_zero_std": 0.6, + "grad_norm": 0.6455156803131104, + "learning_rate": 6.01e-06, + "loss": 0.026773136854171754, + "num_tokens": 1417461.0, + "reward": 0.225, + "reward_std": 0.1632926881313324, + "rewards/boxed_accuracy_reward/mean": 0.225, + "rewards/boxed_accuracy_reward/std": 0.1632926881313324, + "step": 400, + "step_time": 21.00104573589997 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.05, + "completions/max_length": 493.7, + "completions/max_terminated_length": 403.2, + "completions/mean_length": 278.675, + "completions/mean_terminated_length": 238.26178894042968, + "completions/min_length": 146.1, + "completions/min_terminated_length": 146.1, + "entropy": 0.20032316148281099, + "epoch": 0.041, + "frac_reward_zero_std": 0.6, + "grad_norm": 0.0, + "learning_rate": 5.91e-06, + "loss": 0.045339572429656985, + "num_tokens": 1456083.0, + "reward": 0.1375, + "reward_std": 0.15235702097415924, + "rewards/boxed_accuracy_reward/mean": 0.1375, + "rewards/boxed_accuracy_reward/std": 0.15235702097415924, + "step": 410, + "step_time": 33.424314678498924 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0125, + "completions/max_length": 380.4, + "completions/max_terminated_length": 327.8, + "completions/mean_length": 234.2375, + "completions/mean_terminated_length": 225.44107360839843, + "completions/min_length": 155.1, + "completions/min_terminated_length": 155.1, + "entropy": 0.20021430365741252, + "epoch": 0.042, + "frac_reward_zero_std": 0.6, + "grad_norm": 0.0, + "learning_rate": 5.81e-06, + "loss": 0.017741450667381288, + "num_tokens": 1487414.0, + "reward": 0.3375, + "reward_std": 0.1977891743183136, + "rewards/boxed_accuracy_reward/mean": 0.3375, + "rewards/boxed_accuracy_reward/std": 0.19778918027877807, + "step": 420, + "step_time": 25.730258163498366 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 358.2, + "completions/max_terminated_length": 358.2, + "completions/mean_length": 223.7375, + "completions/mean_terminated_length": 223.7375, + "completions/min_length": 126.5, + "completions/min_terminated_length": 126.5, + "entropy": 0.27519515678286555, + "epoch": 0.043, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.0, + "learning_rate": 5.71e-06, + "loss": 0.019644300639629363, + "num_tokens": 1518057.0, + "reward": 0.1375, + "reward_std": 0.19864802658557892, + "rewards/boxed_accuracy_reward/mean": 0.1375, + "rewards/boxed_accuracy_reward/std": 0.19864802658557892, + "step": 430, + "step_time": 24.234244189500167 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 320.9, + "completions/max_terminated_length": 320.9, + "completions/mean_length": 207.3625, + "completions/mean_terminated_length": 207.3625, + "completions/min_length": 140.0, + "completions/min_terminated_length": 140.0, + "entropy": 0.23536108247935772, + "epoch": 0.044, + "frac_reward_zero_std": 0.7, + "grad_norm": 0.0, + "learning_rate": 5.610000000000001e-06, + "loss": 0.01632075011730194, + "num_tokens": 1549726.0, + "reward": 0.2875, + "reward_std": 0.13509859144687653, + "rewards/boxed_accuracy_reward/mean": 0.2875, + "rewards/boxed_accuracy_reward/std": 0.13509859144687653, + "step": 440, + "step_time": 21.775102456000603 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.025, + "completions/max_length": 491.2, + "completions/max_terminated_length": 397.3, + "completions/mean_length": 289.6375, + "completions/mean_terminated_length": 272.23036041259763, + "completions/min_length": 176.8, + "completions/min_terminated_length": 176.8, + "entropy": 0.20252985879778862, + "epoch": 0.045, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.0, + "learning_rate": 5.510000000000001e-06, + "loss": 0.07112041711807252, + "num_tokens": 1586873.0, + "reward": 0.2, + "reward_std": 0.23691109120845794, + "rewards/boxed_accuracy_reward/mean": 0.2, + "rewards/boxed_accuracy_reward/std": 0.23691110908985138, + "step": 450, + "step_time": 33.158058354499374 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 447.2, + "completions/max_terminated_length": 447.2, + "completions/mean_length": 284.3875, + "completions/mean_terminated_length": 284.3875, + "completions/min_length": 166.0, + "completions/min_terminated_length": 166.0, + "entropy": 0.1773521050810814, + "epoch": 0.046, + "frac_reward_zero_std": 0.7, + "grad_norm": 0.0, + "learning_rate": 5.410000000000001e-06, + "loss": -0.04271512031555176, + "num_tokens": 1622272.0, + "reward": 0.25, + "reward_std": 0.13340125977993011, + "rewards/boxed_accuracy_reward/mean": 0.25, + "rewards/boxed_accuracy_reward/std": 0.1334012657403946, + "step": 460, + "step_time": 30.262695012098995 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.025, + "completions/max_length": 510.3, + "completions/max_terminated_length": 417.6, + "completions/mean_length": 271.6625, + "completions/mean_terminated_length": 254.18214416503906, + "completions/min_length": 141.5, + "completions/min_terminated_length": 141.5, + "entropy": 0.23274324238300323, + "epoch": 0.047, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.0, + "learning_rate": 5.310000000000001e-06, + "loss": 0.08956294059753418, + "num_tokens": 1657333.0, + "reward": 0.3, + "reward_std": 0.20411193668842315, + "rewards/boxed_accuracy_reward/mean": 0.3, + "rewards/boxed_accuracy_reward/std": 0.20411194264888763, + "step": 470, + "step_time": 34.52710425489859 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0125, + "completions/max_length": 446.0, + "completions/max_terminated_length": 396.1, + "completions/mean_length": 276.6125, + "completions/mean_terminated_length": 268.86607360839844, + "completions/min_length": 169.2, + "completions/min_terminated_length": 169.2, + "entropy": 0.25105012357234957, + "epoch": 0.048, + "frac_reward_zero_std": 0.8, + "grad_norm": 0.0, + "learning_rate": 5.210000000000001e-06, + "loss": -0.00809616819024086, + "num_tokens": 1694526.0, + "reward": 0.0375, + "reward_std": 0.0816463440656662, + "rewards/boxed_accuracy_reward/mean": 0.0375, + "rewards/boxed_accuracy_reward/std": 0.0816463440656662, + "step": 480, + "step_time": 30.273345093600803 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0125, + "completions/max_length": 442.4, + "completions/max_terminated_length": 381.1, + "completions/mean_length": 224.5125, + "completions/mean_terminated_length": 214.30535736083985, + "completions/min_length": 104.2, + "completions/min_terminated_length": 104.2, + "entropy": 0.24412711150944233, + "epoch": 0.049, + "frac_reward_zero_std": 0.6, + "grad_norm": 0.4479704797267914, + "learning_rate": 5.11e-06, + "loss": 0.03438442945480347, + "num_tokens": 1726799.0, + "reward": 0.2125, + "reward_std": 0.1759178400039673, + "rewards/boxed_accuracy_reward/mean": 0.2125, + "rewards/boxed_accuracy_reward/std": 0.17591784596443177, + "step": 490, + "step_time": 29.914187529801712 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 327.1, + "completions/max_terminated_length": 327.1, + "completions/mean_length": 217.2875, + "completions/mean_terminated_length": 217.2875, + "completions/min_length": 128.0, + "completions/min_terminated_length": 128.0, + "entropy": 0.23657882679253817, + "epoch": 0.05, + "frac_reward_zero_std": 0.6, + "grad_norm": 0.6822343468666077, + "learning_rate": 5.01e-06, + "loss": 0.04140647947788238, + "num_tokens": 1758502.0, + "reward": 0.275, + "reward_std": 0.1632926881313324, + "rewards/boxed_accuracy_reward/mean": 0.275, + "rewards/boxed_accuracy_reward/std": 0.1632926881313324, + "step": 500, + "step_time": 22.14599506190134 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 296.2, + "completions/max_terminated_length": 296.2, + "completions/mean_length": 206.925, + "completions/mean_terminated_length": 206.925, + "completions/min_length": 134.8, + "completions/min_terminated_length": 134.8, + "entropy": 0.21120237074792386, + "epoch": 0.051, + "frac_reward_zero_std": 0.4, + "grad_norm": 0.0, + "learning_rate": 4.9100000000000004e-06, + "loss": -0.02729707658290863, + "num_tokens": 1787704.0, + "reward": 0.4125, + "reward_std": 0.2592615157365799, + "rewards/boxed_accuracy_reward/mean": 0.4125, + "rewards/boxed_accuracy_reward/std": 0.2592615157365799, + "step": 510, + "step_time": 20.121878111200203 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 397.7, + "completions/max_terminated_length": 397.7, + "completions/mean_length": 263.6125, + "completions/mean_terminated_length": 263.6125, + "completions/min_length": 164.6, + "completions/min_terminated_length": 164.6, + "entropy": 0.214491768181324, + "epoch": 0.052, + "frac_reward_zero_std": 0.6, + "grad_norm": 0.0, + "learning_rate": 4.8100000000000005e-06, + "loss": 0.003942453116178512, + "num_tokens": 1822977.0, + "reward": 0.0875, + "reward_std": 0.16875659823417663, + "rewards/boxed_accuracy_reward/mean": 0.0875, + "rewards/boxed_accuracy_reward/std": 0.1687566041946411, + "step": 520, + "step_time": 26.994221128601204 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0375, + "completions/max_length": 485.0, + "completions/max_terminated_length": 358.8, + "completions/mean_length": 280.6625, + "completions/mean_terminated_length": 254.4261932373047, + "completions/min_length": 165.4, + "completions/min_terminated_length": 165.4, + "entropy": 0.3250377386808395, + "epoch": 0.053, + "frac_reward_zero_std": 0.6, + "grad_norm": 0.6389704346656799, + "learning_rate": 4.71e-06, + "loss": 0.08574633002281189, + "num_tokens": 1859638.0, + "reward": 0.15, + "reward_std": 0.15782093107700348, + "rewards/boxed_accuracy_reward/mean": 0.15, + "rewards/boxed_accuracy_reward/std": 0.15782093703746797, + "step": 530, + "step_time": 32.832222134500626 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 361.9, + "completions/max_terminated_length": 361.9, + "completions/mean_length": 252.375, + "completions/mean_terminated_length": 252.375, + "completions/min_length": 164.6, + "completions/min_terminated_length": 164.6, + "entropy": 0.2565031200647354, + "epoch": 0.054, + "frac_reward_zero_std": 0.6, + "grad_norm": 0.0, + "learning_rate": 4.610000000000001e-06, + "loss": 0.021490342915058136, + "num_tokens": 1893300.0, + "reward": 0.1625, + "reward_std": 0.20871699452400208, + "rewards/boxed_accuracy_reward/mean": 0.1625, + "rewards/boxed_accuracy_reward/std": 0.2087170124053955, + "step": 540, + "step_time": 24.475655020699925 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.025, + "completions/max_length": 373.8, + "completions/max_terminated_length": 299.6, + "completions/mean_length": 206.3, + "completions/mean_terminated_length": 185.17916717529297, + "completions/min_length": 111.3, + "completions/min_terminated_length": 111.3, + "entropy": 0.22958396673202514, + "epoch": 0.055, + "frac_reward_zero_std": 0.3, + "grad_norm": 0.6256431937217712, + "learning_rate": 4.510000000000001e-06, + "loss": 0.03615856468677521, + "num_tokens": 1923932.0, + "reward": 0.375, + "reward_std": 0.29838343262672423, + "rewards/boxed_accuracy_reward/mean": 0.375, + "rewards/boxed_accuracy_reward/std": 0.2983834445476532, + "step": 550, + "step_time": 25.349692796901216 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.05, + "completions/max_length": 475.5, + "completions/max_terminated_length": 364.6, + "completions/mean_length": 278.4375, + "completions/mean_terminated_length": 239.52916870117187, + "completions/min_length": 158.6, + "completions/min_terminated_length": 158.6, + "entropy": 0.2459364876151085, + "epoch": 0.056, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.8983960151672363, + "learning_rate": 4.41e-06, + "loss": -0.031027239561080933, + "num_tokens": 1963111.0, + "reward": 0.075, + "reward_std": 0.18771235942840575, + "rewards/boxed_accuracy_reward/mean": 0.075, + "rewards/boxed_accuracy_reward/std": 0.18771235942840575, + "step": 560, + "step_time": 32.03998489129954 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0125, + "completions/max_length": 419.6, + "completions/max_terminated_length": 378.6, + "completions/mean_length": 257.3625, + "completions/mean_terminated_length": 249.33035888671876, + "completions/min_length": 171.2, + "completions/min_terminated_length": 171.2, + "entropy": 0.2526143785566092, + "epoch": 0.057, + "frac_reward_zero_std": 0.7, + "grad_norm": 0.0, + "learning_rate": 4.31e-06, + "loss": 0.01212334781885147, + "num_tokens": 1998908.0, + "reward": 0.2, + "reward_std": 0.14056250154972078, + "rewards/boxed_accuracy_reward/mean": 0.2, + "rewards/boxed_accuracy_reward/std": 0.14056250751018523, + "step": 570, + "step_time": 28.352883074600687 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 394.2, + "completions/max_terminated_length": 394.2, + "completions/mean_length": 257.6875, + "completions/mean_terminated_length": 257.6875, + "completions/min_length": 160.4, + "completions/min_terminated_length": 160.4, + "entropy": 0.2496233806014061, + "epoch": 0.058, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.6807348132133484, + "learning_rate": 4.21e-06, + "loss": 0.0322126567363739, + "num_tokens": 2034387.0, + "reward": 0.1375, + "reward_std": 0.1767766922712326, + "rewards/boxed_accuracy_reward/mean": 0.1375, + "rewards/boxed_accuracy_reward/std": 0.1767766922712326, + "step": 580, + "step_time": 26.78779618450062 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0125, + "completions/max_length": 371.7, + "completions/max_terminated_length": 302.6, + "completions/mean_length": 208.2625, + "completions/mean_terminated_length": 198.36964416503906, + "completions/min_length": 121.9, + "completions/min_terminated_length": 121.9, + "entropy": 0.22029370740056037, + "epoch": 0.059, + "frac_reward_zero_std": 0.6, + "grad_norm": 0.0, + "learning_rate": 4.1100000000000005e-06, + "loss": 0.0259624183177948, + "num_tokens": 2067680.0, + "reward": 0.275, + "reward_std": 0.18685350716114044, + "rewards/boxed_accuracy_reward/mean": 0.275, + "rewards/boxed_accuracy_reward/std": 0.18685351312160492, + "step": 590, + "step_time": 25.24782887190013 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.025, + "completions/max_length": 447.7, + "completions/max_terminated_length": 380.5, + "completions/mean_length": 247.1375, + "completions/mean_terminated_length": 229.69286193847657, + "completions/min_length": 137.8, + "completions/min_terminated_length": 137.8, + "entropy": 0.24857322350144387, + "epoch": 0.06, + "frac_reward_zero_std": 0.7, + "grad_norm": 0.0, + "learning_rate": 4.0100000000000006e-06, + "loss": 0.04133725166320801, + "num_tokens": 2101531.0, + "reward": 0.1, + "reward_std": 0.13340125977993011, + "rewards/boxed_accuracy_reward/mean": 0.1, + "rewards/boxed_accuracy_reward/std": 0.1334012657403946, + "step": 600, + "step_time": 30.182934339900385 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0125, + "completions/max_length": 423.4, + "completions/max_terminated_length": 362.5, + "completions/mean_length": 264.475, + "completions/mean_terminated_length": 255.80714416503906, + "completions/min_length": 171.7, + "completions/min_terminated_length": 171.7, + "entropy": 0.23762509562075138, + "epoch": 0.061, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.8978026509284973, + "learning_rate": 3.910000000000001e-06, + "loss": 0.04321916103363037, + "num_tokens": 2137113.0, + "reward": 0.4, + "reward_std": 0.24030575454235076, + "rewards/boxed_accuracy_reward/mean": 0.4, + "rewards/boxed_accuracy_reward/std": 0.24030576050281524, + "step": 610, + "step_time": 28.61728952409976 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 368.3, + "completions/max_terminated_length": 368.3, + "completions/mean_length": 232.825, + "completions/mean_terminated_length": 232.825, + "completions/min_length": 147.5, + "completions/min_terminated_length": 147.5, + "entropy": 0.2583306774497032, + "epoch": 0.062, + "frac_reward_zero_std": 0.7, + "grad_norm": 0.4808867573738098, + "learning_rate": 3.8100000000000004e-06, + "loss": 0.0032923467457294463, + "num_tokens": 2169507.0, + "reward": 0.0625, + "reward_std": 0.12793734967708587, + "rewards/boxed_accuracy_reward/mean": 0.0625, + "rewards/boxed_accuracy_reward/std": 0.12793734967708587, + "step": 620, + "step_time": 24.657616854002118 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 393.6, + "completions/max_terminated_length": 393.6, + "completions/mean_length": 239.7875, + "completions/mean_terminated_length": 239.7875, + "completions/min_length": 150.1, + "completions/min_terminated_length": 150.1, + "entropy": 0.24895308986306192, + "epoch": 0.063, + "frac_reward_zero_std": 0.6, + "grad_norm": 0.0, + "learning_rate": 3.7100000000000005e-06, + "loss": -0.008826246112585067, + "num_tokens": 2202426.0, + "reward": 0.1625, + "reward_std": 0.19062793254852295, + "rewards/boxed_accuracy_reward/mean": 0.1625, + "rewards/boxed_accuracy_reward/std": 0.19062793850898743, + "step": 630, + "step_time": 26.481925834499272 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0125, + "completions/max_length": 490.3, + "completions/max_terminated_length": 412.0, + "completions/mean_length": 271.6625, + "completions/mean_terminated_length": 261.0321441650391, + "completions/min_length": 156.2, + "completions/min_terminated_length": 156.2, + "entropy": 0.2555754192173481, + "epoch": 0.064, + "frac_reward_zero_std": 0.7, + "grad_norm": 0.0, + "learning_rate": 3.61e-06, + "loss": -0.015103229880332946, + "num_tokens": 2237311.0, + "reward": 0.225, + "reward_std": 0.13340125977993011, + "rewards/boxed_accuracy_reward/mean": 0.225, + "rewards/boxed_accuracy_reward/std": 0.1334012657403946, + "step": 640, + "step_time": 32.94274350120031 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 445.9, + "completions/max_terminated_length": 445.9, + "completions/mean_length": 247.9125, + "completions/mean_terminated_length": 247.9125, + "completions/min_length": 134.0, + "completions/min_terminated_length": 134.0, + "entropy": 0.28245309442281724, + "epoch": 0.065, + "frac_reward_zero_std": 0.6, + "grad_norm": 0.0, + "learning_rate": 3.5100000000000003e-06, + "loss": 0.05236095190048218, + "num_tokens": 2271440.0, + "reward": 0.15, + "reward_std": 0.18685350716114044, + "rewards/boxed_accuracy_reward/mean": 0.15, + "rewards/boxed_accuracy_reward/std": 0.18685351312160492, + "step": 650, + "step_time": 30.18701468149884 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0125, + "completions/max_length": 364.8, + "completions/max_terminated_length": 317.3, + "completions/mean_length": 223.35, + "completions/mean_terminated_length": 214.17857360839844, + "completions/min_length": 135.9, + "completions/min_terminated_length": 135.9, + "entropy": 0.265711085498333, + "epoch": 0.066, + "frac_reward_zero_std": 0.3, + "grad_norm": 0.8590646982192993, + "learning_rate": 3.4100000000000004e-06, + "loss": 0.03245477676391602, + "num_tokens": 2304332.0, + "reward": 0.3375, + "reward_std": 0.3311904340982437, + "rewards/boxed_accuracy_reward/mean": 0.3375, + "rewards/boxed_accuracy_reward/std": 0.3311904460191727, + "step": 660, + "step_time": 24.758795369901055 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 369.7, + "completions/max_terminated_length": 369.7, + "completions/mean_length": 255.3375, + "completions/mean_terminated_length": 255.3375, + "completions/min_length": 146.1, + "completions/min_terminated_length": 146.1, + "entropy": 0.2967137709259987, + "epoch": 0.067, + "frac_reward_zero_std": 0.8, + "grad_norm": 0.8069291710853577, + "learning_rate": 3.3100000000000005e-06, + "loss": 0.022205421328544618, + "num_tokens": 2338295.0, + "reward": 0.2, + "reward_std": 0.10350983142852783, + "rewards/boxed_accuracy_reward/mean": 0.2, + "rewards/boxed_accuracy_reward/std": 0.10350984334945679, + "step": 670, + "step_time": 24.84447250970188 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 378.8, + "completions/max_terminated_length": 378.8, + "completions/mean_length": 223.475, + "completions/mean_terminated_length": 223.475, + "completions/min_length": 140.8, + "completions/min_terminated_length": 140.8, + "entropy": 0.3041005529463291, + "epoch": 0.068, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.0, + "learning_rate": 3.21e-06, + "loss": -0.020687612891197204, + "num_tokens": 2369605.0, + "reward": 0.2, + "reward_std": 0.24954409003257752, + "rewards/boxed_accuracy_reward/mean": 0.2, + "rewards/boxed_accuracy_reward/std": 0.24954410195350646, + "step": 680, + "step_time": 25.69439109330124 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 346.6, + "completions/max_terminated_length": 346.6, + "completions/mean_length": 237.9625, + "completions/mean_terminated_length": 237.9625, + "completions/min_length": 147.4, + "completions/min_terminated_length": 147.4, + "entropy": 0.24550328701734542, + "epoch": 0.069, + "frac_reward_zero_std": 0.7, + "grad_norm": 0.9554150700569153, + "learning_rate": 3.1100000000000003e-06, + "loss": 0.013637937605381012, + "num_tokens": 2401898.0, + "reward": 0.1875, + "reward_std": 0.13509859144687653, + "rewards/boxed_accuracy_reward/mean": 0.1875, + "rewards/boxed_accuracy_reward/std": 0.13509859144687653, + "step": 690, + "step_time": 23.408739431599678 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 390.2, + "completions/max_terminated_length": 390.2, + "completions/mean_length": 242.8125, + "completions/mean_terminated_length": 242.8125, + "completions/min_length": 145.6, + "completions/min_terminated_length": 145.6, + "entropy": 0.2589107438921928, + "epoch": 0.07, + "frac_reward_zero_std": 0.6, + "grad_norm": 1.1996217966079712, + "learning_rate": 3.01e-06, + "loss": 0.05301226377487182, + "num_tokens": 2436907.0, + "reward": 0.1125, + "reward_std": 0.18138959705829621, + "rewards/boxed_accuracy_reward/mean": 0.1125, + "rewards/boxed_accuracy_reward/std": 0.18138959705829621, + "step": 700, + "step_time": 26.322794716500358 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0125, + "completions/max_length": 369.9, + "completions/max_terminated_length": 291.4, + "completions/mean_length": 211.5125, + "completions/mean_terminated_length": 200.55178680419922, + "completions/min_length": 133.0, + "completions/min_terminated_length": 133.0, + "entropy": 0.19192061349749565, + "epoch": 0.071, + "frac_reward_zero_std": 0.9, + "grad_norm": 0.0, + "learning_rate": 2.91e-06, + "loss": -0.003464340791106224, + "num_tokens": 2467292.0, + "reward": 0.2875, + "reward_std": 0.03535533845424652, + "rewards/boxed_accuracy_reward/mean": 0.2875, + "rewards/boxed_accuracy_reward/std": 0.03535533845424652, + "step": 710, + "step_time": 25.09874547720101 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.05, + "completions/max_length": 574.9, + "completions/max_terminated_length": 388.2, + "completions/mean_length": 293.5125, + "completions/mean_terminated_length": 260.60000762939455, + "completions/min_length": 169.5, + "completions/min_terminated_length": 169.5, + "entropy": 0.22459689602255822, + "epoch": 0.072, + "frac_reward_zero_std": 0.6, + "grad_norm": 0.2952307164669037, + "learning_rate": 2.8100000000000006e-06, + "loss": 0.04532091617584229, + "num_tokens": 2502805.0, + "reward": 0.25, + "reward_std": 0.15782093107700348, + "rewards/boxed_accuracy_reward/mean": 0.25, + "rewards/boxed_accuracy_reward/std": 0.15782093703746797, + "step": 720, + "step_time": 38.782466992000266 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 315.9, + "completions/max_terminated_length": 315.9, + "completions/mean_length": 216.875, + "completions/mean_terminated_length": 216.875, + "completions/min_length": 130.2, + "completions/min_terminated_length": 130.2, + "entropy": 0.26642010658979415, + "epoch": 0.073, + "frac_reward_zero_std": 0.6, + "grad_norm": 0.7213253974914551, + "learning_rate": 2.7100000000000003e-06, + "loss": -0.004600425064563751, + "num_tokens": 2535131.0, + "reward": 0.1375, + "reward_std": 0.17422835528850555, + "rewards/boxed_accuracy_reward/mean": 0.1375, + "rewards/boxed_accuracy_reward/std": 0.17422835528850555, + "step": 730, + "step_time": 21.296572963699145 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 427.0, + "completions/max_terminated_length": 427.0, + "completions/mean_length": 246.8375, + "completions/mean_terminated_length": 246.8375, + "completions/min_length": 159.4, + "completions/min_terminated_length": 159.4, + "entropy": 0.2320882935076952, + "epoch": 0.074, + "frac_reward_zero_std": 0.6, + "grad_norm": 0.0, + "learning_rate": 2.6100000000000004e-06, + "loss": -0.0016866859048604965, + "num_tokens": 2567462.0, + "reward": 0.225, + "reward_std": 0.1796922653913498, + "rewards/boxed_accuracy_reward/mean": 0.225, + "rewards/boxed_accuracy_reward/std": 0.17969227135181426, + "step": 740, + "step_time": 28.78491548419988 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0125, + "completions/max_length": 407.3, + "completions/max_terminated_length": 341.9, + "completions/mean_length": 229.175, + "completions/mean_terminated_length": 218.69285736083984, + "completions/min_length": 129.1, + "completions/min_terminated_length": 129.1, + "entropy": 0.2695804834365845, + "epoch": 0.075, + "frac_reward_zero_std": 0.6, + "grad_norm": 1.118378758430481, + "learning_rate": 2.51e-06, + "loss": 0.02082826942205429, + "num_tokens": 2601372.0, + "reward": 0.175, + "reward_std": 0.17045392990112304, + "rewards/boxed_accuracy_reward/mean": 0.175, + "rewards/boxed_accuracy_reward/std": 0.17045392990112304, + "step": 750, + "step_time": 27.563064585998653 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 250.8, + "completions/max_terminated_length": 250.8, + "completions/mean_length": 172.0375, + "completions/mean_terminated_length": 172.0375, + "completions/min_length": 121.6, + "completions/min_terminated_length": 121.6, + "entropy": 0.2495213456451893, + "epoch": 0.076, + "frac_reward_zero_std": 0.7, + "grad_norm": 1.1198997497558594, + "learning_rate": 2.4100000000000002e-06, + "loss": -0.0004969652742147446, + "num_tokens": 2631303.0, + "reward": 0.3125, + "reward_std": 0.14433692693710326, + "rewards/boxed_accuracy_reward/mean": 0.3125, + "rewards/boxed_accuracy_reward/std": 0.14433693289756774, + "step": 760, + "step_time": 17.046090553899557 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 341.0, + "completions/max_terminated_length": 341.0, + "completions/mean_length": 209.725, + "completions/mean_terminated_length": 209.725, + "completions/min_length": 120.9, + "completions/min_terminated_length": 120.9, + "entropy": 0.2746701970696449, + "epoch": 0.077, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.0, + "learning_rate": 2.3100000000000003e-06, + "loss": 0.04669084250926971, + "num_tokens": 2661489.0, + "reward": 0.25, + "reward_std": 0.23691109120845794, + "rewards/boxed_accuracy_reward/mean": 0.25, + "rewards/boxed_accuracy_reward/std": 0.23691110908985138, + "step": 770, + "step_time": 23.079953304801894 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 599.1, + "completions/max_terminated_length": 445.5, + "completions/mean_length": 323.7625, + "completions/mean_terminated_length": 287.5857177734375, + "completions/min_length": 165.4, + "completions/min_terminated_length": 165.4, + "entropy": 0.20161464884877206, + "epoch": 0.078, + "frac_reward_zero_std": 0.7, + "grad_norm": 0.6654395461082458, + "learning_rate": 2.21e-06, + "loss": -0.01412966102361679, + "num_tokens": 2703534.0, + "reward": 0.2, + "reward_std": 0.14056250154972078, + "rewards/boxed_accuracy_reward/mean": 0.2, + "rewards/boxed_accuracy_reward/std": 0.14056250751018523, + "step": 780, + "step_time": 40.28982557420022 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.025, + "completions/max_length": 488.7, + "completions/max_terminated_length": 422.1, + "completions/mean_length": 281.3625, + "completions/mean_terminated_length": 261.6958343505859, + "completions/min_length": 144.9, + "completions/min_terminated_length": 144.9, + "entropy": 0.23346679359674455, + "epoch": 0.079, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.0, + "learning_rate": 2.11e-06, + "loss": 0.03522632122039795, + "num_tokens": 2739035.0, + "reward": 0.25, + "reward_std": 0.2529387533664703, + "rewards/boxed_accuracy_reward/mean": 0.25, + "rewards/boxed_accuracy_reward/std": 0.2529387533664703, + "step": 790, + "step_time": 33.08498501170034 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 344.0, + "completions/max_terminated_length": 344.0, + "completions/mean_length": 210.85, + "completions/mean_terminated_length": 210.85, + "completions/min_length": 133.1, + "completions/min_terminated_length": 133.1, + "entropy": 0.25274474173784256, + "epoch": 0.08, + "frac_reward_zero_std": 0.7, + "grad_norm": 0.33814314007759094, + "learning_rate": 2.0100000000000002e-06, + "loss": 0.009773983806371688, + "num_tokens": 2768951.0, + "reward": 0.2, + "reward_std": 0.13340125977993011, + "rewards/boxed_accuracy_reward/mean": 0.2, + "rewards/boxed_accuracy_reward/std": 0.1334012657403946, + "step": 800, + "step_time": 23.216696536900418 + } + ], + "logging_steps": 10, + "max_steps": 1000, + "num_input_tokens_seen": 2768951, + "num_train_epochs": 1, + "save_steps": 200, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 0.0, + "train_batch_size": 8, + "trial_name": null, + "trial_params": null +} diff --git a/training_args.bin b/training_args.bin new file mode 100644 index 0000000..763f244 --- /dev/null +++ b/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4d1dfbce22d8487292784256a5f8170a87f2952c85a6741795b7e2c965f21f1c +size 7185