From 5a08ed1b76ea9de39b0053abeea5193521c094c4 Mon Sep 17 00:00:00 2001 From: ModelHub XC Date: Fri, 21 Aug 2026 04:37:16 +0800 Subject: [PATCH] =?UTF-8?q?=E5=88=9D=E5=A7=8B=E5=8C=96=E9=A1=B9=E7=9B=AE?= =?UTF-8?q?=EF=BC=8C=E7=94=B1ModelHub=20XC=E7=A4=BE=E5=8C=BA=E6=8F=90?= =?UTF-8?q?=E4=BE=9B=E6=A8=A1=E5=9E=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Model: kangdawei/MMR-Sigmoid-DAPO-8B Source: Original Platform --- .gitattributes | 37 + README.md | 70 + adapter/README.md | 209 + adapter/adapter_config.json | 46 + adapter/adapter_model.safetensors | 3 + adapter/chat_template.jinja | 1 + adapter/special_tokens_map.json | 23 + adapter/tokenizer.json | 3 + adapter/tokenizer_config.json | 2066 +++++ adapter/training_args.bin | 3 + adapter_config.json | 46 + adapter_model.safetensors | 3 + all_results.json | 8 + chat_template.jinja | 1 + config.json | 35 + generation_config.json | 9 + model-00001-of-00004.safetensors | 3 + model-00002-of-00004.safetensors | 3 + model-00003-of-00004.safetensors | 3 + model-00004-of-00004.safetensors | 3 + model.safetensors.index.json | 299 + special_tokens_map.json | 23 + tokenizer.json | 3 + tokenizer_config.json | 2066 +++++ train_results.json | 8 + trainer_state.json | 11543 ++++++++++++++++++++++++++++ training_args.bin | 3 + 27 files changed, 16520 insertions(+) create mode 100644 .gitattributes create mode 100644 README.md create mode 100644 adapter/README.md create mode 100644 adapter/adapter_config.json create mode 100644 adapter/adapter_model.safetensors create mode 100644 adapter/chat_template.jinja create mode 100644 adapter/special_tokens_map.json create mode 100644 adapter/tokenizer.json create mode 100644 adapter/tokenizer_config.json create mode 100644 adapter/training_args.bin create mode 100644 adapter_config.json create mode 100644 adapter_model.safetensors create mode 100644 all_results.json create mode 100644 chat_template.jinja create mode 100644 config.json create mode 100644 generation_config.json create mode 100644 model-00001-of-00004.safetensors create mode 100644 model-00002-of-00004.safetensors create mode 100644 model-00003-of-00004.safetensors create mode 100644 model-00004-of-00004.safetensors create mode 100644 model.safetensors.index.json create mode 100644 special_tokens_map.json create mode 100644 tokenizer.json create mode 100644 tokenizer_config.json create mode 100644 train_results.json create mode 100644 trainer_state.json create mode 100644 training_args.bin diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..b19d564 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,37 @@ +*.7z filter=lfs diff=lfs merge=lfs -text +*.arrow filter=lfs diff=lfs merge=lfs -text +*.bin filter=lfs diff=lfs merge=lfs -text +*.bz2 filter=lfs diff=lfs merge=lfs -text +*.ckpt filter=lfs diff=lfs merge=lfs -text +*.ftz filter=lfs diff=lfs merge=lfs -text +*.gz filter=lfs diff=lfs merge=lfs -text +*.h5 filter=lfs diff=lfs merge=lfs -text +*.joblib filter=lfs diff=lfs merge=lfs -text +*.lfs.* filter=lfs diff=lfs merge=lfs -text +*.mlmodel filter=lfs diff=lfs merge=lfs -text +*.model filter=lfs diff=lfs merge=lfs -text +*.msgpack filter=lfs diff=lfs merge=lfs -text +*.npy filter=lfs diff=lfs merge=lfs -text +*.npz filter=lfs diff=lfs merge=lfs -text +*.onnx filter=lfs diff=lfs merge=lfs -text +*.ot filter=lfs diff=lfs merge=lfs -text +*.parquet filter=lfs diff=lfs merge=lfs -text +*.pb filter=lfs diff=lfs merge=lfs -text +*.pickle filter=lfs diff=lfs merge=lfs -text +*.pkl filter=lfs diff=lfs merge=lfs -text +*.pt filter=lfs diff=lfs merge=lfs -text +*.pth filter=lfs diff=lfs merge=lfs -text +*.rar filter=lfs diff=lfs merge=lfs -text +*.safetensors filter=lfs diff=lfs merge=lfs -text +saved_model/**/* filter=lfs diff=lfs merge=lfs -text +*.tar.* filter=lfs diff=lfs merge=lfs -text +*.tar filter=lfs diff=lfs merge=lfs -text +*.tflite filter=lfs diff=lfs merge=lfs -text +*.tgz filter=lfs diff=lfs merge=lfs -text +*.wasm filter=lfs diff=lfs merge=lfs -text +*.xz filter=lfs diff=lfs merge=lfs -text +*.zip filter=lfs diff=lfs merge=lfs -text +*.zst filter=lfs diff=lfs merge=lfs -text +*tfevents* filter=lfs diff=lfs merge=lfs -text +tokenizer.json filter=lfs diff=lfs merge=lfs -text +adapter/tokenizer.json filter=lfs diff=lfs merge=lfs -text diff --git a/README.md b/README.md new file mode 100644 index 0000000..7e89268 --- /dev/null +++ b/README.md @@ -0,0 +1,70 @@ +--- +base_model: deepseek-ai/DeepSeek-R1-Distill-Llama-8B +datasets: knoveleng/open-rs +library_name: transformers +model_name: MMR-Sigmoid-DAPO-8B +tags: +- generated_from_trainer +- open-r1 +- dapo +- trl +licence: license +--- + +# Model Card for MMR-Sigmoid-DAPO-8B + +This model is a fine-tuned version of [deepseek-ai/DeepSeek-R1-Distill-Llama-8B](https://huggingface.co/deepseek-ai/DeepSeek-R1-Distill-Llama-8B) on the [knoveleng/open-rs](https://huggingface.co/datasets/knoveleng/open-rs) dataset. +It has been trained using [TRL](https://github.com/huggingface/trl). + +## Quick start + +```python +from transformers import pipeline + +question = "If you had a time machine, but could only go to the past or the future once and never return, which would you choose and why?" +generator = pipeline("text-generation", model="kangdawei/MMR-Sigmoid-DAPO-8B", device="cuda") +output = generator([{"role": "user", "content": question}], max_new_tokens=128, return_full_text=False)[0] +print(output["generated_text"]) +``` + +## Training procedure + + + + +This model was trained with DAPO, a method introduced in [DAPO: An Open-Source LLM Reinforcement Learning System at Scale](https://huggingface.co/papers/2503.14476). + +### Framework versions + +- TRL: 0.16.0.dev0 +- Transformers: 4.57.1 +- Pytorch: 2.5.1 +- Datasets: 3.2.0 +- Tokenizers: 0.22.1 + +## Citations + +Cite DAPO as: + +```bibtex +@article{yu2025dapo, + title = {{DAPO: An Open-Source LLM Reinforcement Learning System at Scale}}, + author = {Qiying Yu and Zheng Zhang and others}, + year = 2025, + eprint = {arXiv:2503.14476}, +} + +``` + +Cite TRL as: + +```bibtex +@misc{vonwerra2022trl, + title = {{TRL: Transformer Reinforcement Learning}}, + author = {Leandro von Werra and Younes Belkada and Lewis Tunstall and Edward Beeching and Tristan Thrush and Nathan Lambert and Shengyi Huang and Kashif Rasul and Quentin Gallouédec}, + year = 2020, + journal = {GitHub repository}, + publisher = {GitHub}, + howpublished = {\url{https://github.com/huggingface/trl}} +} +``` \ No newline at end of file diff --git a/adapter/README.md b/adapter/README.md new file mode 100644 index 0000000..da11736 --- /dev/null +++ b/adapter/README.md @@ -0,0 +1,209 @@ +--- +base_model: deepseek-ai/DeepSeek-R1-Distill-Llama-8B +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:deepseek-ai/DeepSeek-R1-Distill-Llama-8B +- dapo +- lora +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.18.0 \ No newline at end of file diff --git a/adapter/adapter_config.json b/adapter/adapter_config.json new file mode 100644 index 0000000..2eb113c --- /dev/null +++ b/adapter/adapter_config.json @@ -0,0 +1,46 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "deepseek-ai/DeepSeek-R1-Distill-Llama-8B", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 128, + "lora_bias": false, + "lora_dropout": 0.05, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.18.0", + "qalora_group_size": 16, + "r": 64, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "down_proj", + "o_proj", + "q_proj", + "k_proj", + "v_proj", + "up_proj", + "gate_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/adapter/adapter_model.safetensors b/adapter/adapter_model.safetensors new file mode 100644 index 0000000..672b1d3 --- /dev/null +++ b/adapter/adapter_model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8c2ae27d983509d3cf4925056938e5833b17940ee98f48cdeb1c6c45cc2b904e +size 335605144 diff --git a/adapter/chat_template.jinja b/adapter/chat_template.jinja new file mode 100644 index 0000000..c2066bd --- /dev/null +++ b/adapter/chat_template.jinja @@ -0,0 +1 @@ +{% if not add_generation_prompt is defined %}{% set add_generation_prompt = false %}{% endif %}{% set ns = namespace(is_first=false, is_tool=false, is_output_first=true, system_prompt='') %}{%- for message in messages %}{%- if message['role'] == 'system' %}{% set ns.system_prompt = message['content'] %}{%- endif %}{%- endfor %}{{bos_token}}{{ns.system_prompt}}{%- for message in messages %}{%- if message['role'] == 'user' %}{%- set ns.is_tool = false -%}{{'<｜User｜>' + message['content']}}{%- endif %}{%- if message['role'] == 'assistant' and message['content'] is none %}{%- set ns.is_tool = false -%}{%- for tool in message['tool_calls']%}{%- if not ns.is_first %}{{'<｜Assistant｜><｜tool▁calls▁begin｜><｜tool▁call▁begin｜>' + tool['type'] + '<｜tool▁sep｜>' + tool['function']['name'] + '\n' + '```json' + '\n' + tool['function']['arguments'] + '\n' + '```' + '<｜tool▁call▁end｜>'}}{%- set ns.is_first = true -%}{%- else %}{{'\n' + '<｜tool▁call▁begin｜>' + tool['type'] + '<｜tool▁sep｜>' + tool['function']['name'] + '\n' + '```json' + '\n' + tool['function']['arguments'] + '\n' + '```' + '<｜tool▁call▁end｜>'}}{{'<｜tool▁calls▁end｜><｜end▁of▁sentence｜>'}}{%- endif %}{%- endfor %}{%- endif %}{%- if message['role'] == 'assistant' and message['content'] is not none %}{%- if ns.is_tool %}{{'<｜tool▁outputs▁end｜>' + message['content'] + '<｜end▁of▁sentence｜>'}}{%- set ns.is_tool = false -%}{%- else %}{% set content = message['content'] %}{% if '' in content %}{% set content = content.split('')[-1] %}{% endif %}{{'<｜Assistant｜>' + content + '<｜end▁of▁sentence｜>'}}{%- endif %}{%- endif %}{%- if message['role'] == 'tool' %}{%- set ns.is_tool = true -%}{%- if ns.is_output_first %}{{'<｜tool▁outputs▁begin｜><｜tool▁output▁begin｜>' + message['content'] + '<｜tool▁output▁end｜>'}}{%- set ns.is_output_first = false %}{%- else %}{{'\n<｜tool▁output▁begin｜>' + message['content'] + '<｜tool▁output▁end｜>'}}{%- endif %}{%- endif %}{%- endfor -%}{% if ns.is_tool %}{{'<｜tool▁outputs▁end｜>'}}{% endif %}{% if add_generation_prompt and not ns.is_tool %}{{'<｜Assistant｜>\n'}}{% endif %} \ No newline at end of file diff --git a/adapter/special_tokens_map.json b/adapter/special_tokens_map.json new file mode 100644 index 0000000..1d385d6 --- /dev/null +++ b/adapter/special_tokens_map.json @@ -0,0 +1,23 @@ +{ + "bos_token": { + "content": "<｜begin▁of▁sentence｜>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "eos_token": { + "content": "<｜end▁of▁sentence｜>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "pad_token": { + "content": "<｜end▁of▁sentence｜>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + } +} diff --git a/adapter/tokenizer.json b/adapter/tokenizer.json new file mode 100644 index 0000000..e77a163 --- /dev/null +++ b/adapter/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:26c881aaf4ef935b1516ec79ad6405dd2a459f2b5d431a8a4a1399c92f3ba022 +size 17209711 diff --git a/adapter/tokenizer_config.json b/adapter/tokenizer_config.json new file mode 100644 index 0000000..dd34db6 --- /dev/null +++ b/adapter/tokenizer_config.json @@ -0,0 +1,2066 @@ +{ + "add_bos_token": true, + "add_eos_token": false, + "add_prefix_space": null, + "added_tokens_decoder": { + "128000": { + "content": "<｜begin▁of▁sentence｜>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128001": { + "content": "<｜end▁of▁sentence｜>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128002": { + "content": "<|reserved_special_token_0|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128003": { + "content": "<|reserved_special_token_1|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128004": { + "content": "<|finetune_right_pad_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128005": { + "content": "<|reserved_special_token_2|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128006": { + "content": "<|start_header_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128007": { + "content": "<|end_header_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128008": { + "content": "<|eom_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128009": { + "content": "<|eot_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128010": { + "content": "<|python_tag|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128011": { + "content": "<｜User｜>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "128012": { + "content": "<｜Assistant｜>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "128013": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "128014": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "128015": { + "content": "<｜▁pad▁｜>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128016": { + "content": "<|reserved_special_token_8|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128017": { + "content": "<|reserved_special_token_9|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128018": { + "content": "<|reserved_special_token_10|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128019": { + "content": "<|reserved_special_token_11|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128020": { + "content": "<|reserved_special_token_12|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128021": { + "content": "<|reserved_special_token_13|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128022": { + "content": "<|reserved_special_token_14|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128023": { + "content": "<|reserved_special_token_15|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128024": { + "content": "<|reserved_special_token_16|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128025": { + "content": "<|reserved_special_token_17|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128026": { + "content": "<|reserved_special_token_18|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128027": { + "content": "<|reserved_special_token_19|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128028": { + "content": "<|reserved_special_token_20|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128029": { + "content": "<|reserved_special_token_21|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128030": { + "content": "<|reserved_special_token_22|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128031": { + "content": "<|reserved_special_token_23|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128032": { + "content": "<|reserved_special_token_24|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128033": { + "content": "<|reserved_special_token_25|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128034": { + "content": "<|reserved_special_token_26|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128035": { + "content": "<|reserved_special_token_27|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128036": { + "content": "<|reserved_special_token_28|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128037": { + "content": "<|reserved_special_token_29|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128038": { + "content": "<|reserved_special_token_30|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128039": { + "content": "<|reserved_special_token_31|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128040": { + "content": "<|reserved_special_token_32|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128041": { + "content": "<|reserved_special_token_33|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128042": { + "content": "<|reserved_special_token_34|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128043": { + "content": "<|reserved_special_token_35|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128044": { + "content": "<|reserved_special_token_36|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128045": { + "content": "<|reserved_special_token_37|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128046": { + "content": "<|reserved_special_token_38|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128047": { + "content": "<|reserved_special_token_39|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128048": { + "content": "<|reserved_special_token_40|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128049": { + "content": "<|reserved_special_token_41|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128050": { + "content": "<|reserved_special_token_42|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128051": { + "content": "<|reserved_special_token_43|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128052": { + "content": "<|reserved_special_token_44|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128053": { + "content": "<|reserved_special_token_45|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128054": { + "content": "<|reserved_special_token_46|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128055": { + "content": "<|reserved_special_token_47|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128056": { + "content": "<|reserved_special_token_48|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128057": { + "content": "<|reserved_special_token_49|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128058": { + "content": "<|reserved_special_token_50|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128059": { + "content": "<|reserved_special_token_51|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128060": { + "content": "<|reserved_special_token_52|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128061": { + "content": "<|reserved_special_token_53|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128062": { + "content": "<|reserved_special_token_54|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128063": { + "content": "<|reserved_special_token_55|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128064": { + "content": "<|reserved_special_token_56|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128065": { + "content": "<|reserved_special_token_57|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128066": { + "content": "<|reserved_special_token_58|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128067": { + "content": "<|reserved_special_token_59|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128068": { + "content": "<|reserved_special_token_60|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128069": { + "content": "<|reserved_special_token_61|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128070": { + "content": "<|reserved_special_token_62|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128071": { + "content": "<|reserved_special_token_63|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128072": { + "content": "<|reserved_special_token_64|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128073": { + "content": "<|reserved_special_token_65|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128074": { + "content": "<|reserved_special_token_66|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128075": { + "content": "<|reserved_special_token_67|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128076": { + "content": "<|reserved_special_token_68|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128077": { + "content": "<|reserved_special_token_69|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128078": { + "content": "<|reserved_special_token_70|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128079": { + "content": "<|reserved_special_token_71|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128080": { + "content": "<|reserved_special_token_72|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128081": { + "content": "<|reserved_special_token_73|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128082": { + "content": "<|reserved_special_token_74|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128083": { + "content": "<|reserved_special_token_75|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128084": { + "content": "<|reserved_special_token_76|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128085": { + "content": "<|reserved_special_token_77|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128086": { + "content": "<|reserved_special_token_78|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128087": { + "content": "<|reserved_special_token_79|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128088": { + "content": "<|reserved_special_token_80|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128089": { + "content": "<|reserved_special_token_81|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128090": { + "content": "<|reserved_special_token_82|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128091": { + "content": "<|reserved_special_token_83|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128092": { + "content": "<|reserved_special_token_84|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128093": { + "content": "<|reserved_special_token_85|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128094": { + "content": "<|reserved_special_token_86|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128095": { + "content": "<|reserved_special_token_87|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128096": { + "content": "<|reserved_special_token_88|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128097": { + "content": "<|reserved_special_token_89|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128098": { + "content": "<|reserved_special_token_90|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128099": { + "content": "<|reserved_special_token_91|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128100": { + "content": "<|reserved_special_token_92|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128101": { + "content": "<|reserved_special_token_93|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128102": { + "content": "<|reserved_special_token_94|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128103": { + "content": "<|reserved_special_token_95|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128104": { + "content": "<|reserved_special_token_96|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128105": { + "content": "<|reserved_special_token_97|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128106": { + "content": "<|reserved_special_token_98|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128107": { + "content": "<|reserved_special_token_99|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128108": { + "content": "<|reserved_special_token_100|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128109": { + "content": "<|reserved_special_token_101|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128110": { + "content": "<|reserved_special_token_102|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128111": { + "content": "<|reserved_special_token_103|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128112": { + "content": "<|reserved_special_token_104|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128113": { + "content": "<|reserved_special_token_105|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128114": { + "content": "<|reserved_special_token_106|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128115": { + "content": "<|reserved_special_token_107|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128116": { + "content": "<|reserved_special_token_108|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128117": { + "content": "<|reserved_special_token_109|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128118": { + "content": "<|reserved_special_token_110|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128119": { + "content": "<|reserved_special_token_111|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128120": { + "content": "<|reserved_special_token_112|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128121": { + "content": "<|reserved_special_token_113|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128122": { + "content": "<|reserved_special_token_114|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128123": { + "content": "<|reserved_special_token_115|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128124": { + "content": "<|reserved_special_token_116|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128125": { + "content": "<|reserved_special_token_117|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128126": { + "content": "<|reserved_special_token_118|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128127": { + "content": "<|reserved_special_token_119|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128128": { + "content": "<|reserved_special_token_120|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128129": { + "content": "<|reserved_special_token_121|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128130": { + "content": "<|reserved_special_token_122|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128131": { + "content": "<|reserved_special_token_123|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128132": { + "content": "<|reserved_special_token_124|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128133": { + "content": "<|reserved_special_token_125|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128134": { + "content": "<|reserved_special_token_126|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128135": { + "content": "<|reserved_special_token_127|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128136": { + "content": "<|reserved_special_token_128|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128137": { + "content": "<|reserved_special_token_129|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128138": { + "content": "<|reserved_special_token_130|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128139": { + "content": "<|reserved_special_token_131|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128140": { + "content": "<|reserved_special_token_132|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128141": { + "content": "<|reserved_special_token_133|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128142": { + "content": "<|reserved_special_token_134|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128143": { + "content": "<|reserved_special_token_135|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128144": { + "content": "<|reserved_special_token_136|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128145": { + "content": "<|reserved_special_token_137|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128146": { + "content": "<|reserved_special_token_138|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128147": { + "content": "<|reserved_special_token_139|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128148": { + "content": "<|reserved_special_token_140|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128149": { + "content": "<|reserved_special_token_141|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128150": { + "content": "<|reserved_special_token_142|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128151": { + "content": "<|reserved_special_token_143|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128152": { + "content": "<|reserved_special_token_144|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128153": { + "content": "<|reserved_special_token_145|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128154": { + "content": "<|reserved_special_token_146|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128155": { + "content": "<|reserved_special_token_147|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128156": { + "content": "<|reserved_special_token_148|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128157": { + "content": "<|reserved_special_token_149|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128158": { + "content": "<|reserved_special_token_150|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128159": { + "content": "<|reserved_special_token_151|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128160": { + "content": "<|reserved_special_token_152|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128161": { + "content": "<|reserved_special_token_153|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128162": { + "content": "<|reserved_special_token_154|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128163": { + "content": "<|reserved_special_token_155|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128164": { + "content": "<|reserved_special_token_156|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128165": { + "content": "<|reserved_special_token_157|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128166": { + "content": "<|reserved_special_token_158|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128167": { + "content": "<|reserved_special_token_159|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128168": { + "content": "<|reserved_special_token_160|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128169": { + "content": "<|reserved_special_token_161|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128170": { + "content": "<|reserved_special_token_162|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128171": { + "content": "<|reserved_special_token_163|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128172": { + "content": "<|reserved_special_token_164|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128173": { + "content": "<|reserved_special_token_165|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128174": { + "content": "<|reserved_special_token_166|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128175": { + "content": "<|reserved_special_token_167|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128176": { + "content": "<|reserved_special_token_168|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128177": { + "content": "<|reserved_special_token_169|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128178": { + "content": "<|reserved_special_token_170|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128179": { + "content": "<|reserved_special_token_171|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128180": { + "content": "<|reserved_special_token_172|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128181": { + "content": "<|reserved_special_token_173|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128182": { + "content": "<|reserved_special_token_174|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128183": { + "content": "<|reserved_special_token_175|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128184": { + "content": "<|reserved_special_token_176|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128185": { + "content": "<|reserved_special_token_177|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128186": { + "content": "<|reserved_special_token_178|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128187": { + "content": "<|reserved_special_token_179|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128188": { + "content": "<|reserved_special_token_180|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128189": { + "content": "<|reserved_special_token_181|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128190": { + "content": "<|reserved_special_token_182|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128191": { + "content": "<|reserved_special_token_183|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128192": { + "content": "<|reserved_special_token_184|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128193": { + "content": "<|reserved_special_token_185|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128194": { + "content": "<|reserved_special_token_186|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128195": { + "content": "<|reserved_special_token_187|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128196": { + "content": "<|reserved_special_token_188|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128197": { + "content": "<|reserved_special_token_189|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128198": { + "content": "<|reserved_special_token_190|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128199": { + "content": "<|reserved_special_token_191|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128200": { + "content": "<|reserved_special_token_192|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128201": { + "content": "<|reserved_special_token_193|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128202": { + "content": "<|reserved_special_token_194|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128203": { + "content": "<|reserved_special_token_195|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128204": { + "content": "<|reserved_special_token_196|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128205": { + "content": "<|reserved_special_token_197|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128206": { + "content": "<|reserved_special_token_198|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128207": { + "content": "<|reserved_special_token_199|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128208": { + "content": "<|reserved_special_token_200|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128209": { + "content": "<|reserved_special_token_201|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128210": { + "content": "<|reserved_special_token_202|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128211": { + "content": "<|reserved_special_token_203|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128212": { + "content": "<|reserved_special_token_204|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128213": { + "content": "<|reserved_special_token_205|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128214": { + "content": "<|reserved_special_token_206|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128215": { + "content": "<|reserved_special_token_207|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128216": { + "content": "<|reserved_special_token_208|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128217": { + "content": "<|reserved_special_token_209|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128218": { + "content": "<|reserved_special_token_210|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128219": { + "content": "<|reserved_special_token_211|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128220": { + "content": "<|reserved_special_token_212|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128221": { + "content": "<|reserved_special_token_213|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128222": { + "content": "<|reserved_special_token_214|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128223": { + "content": "<|reserved_special_token_215|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128224": { + "content": "<|reserved_special_token_216|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128225": { + "content": "<|reserved_special_token_217|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128226": { + "content": "<|reserved_special_token_218|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128227": { + "content": "<|reserved_special_token_219|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128228": { + "content": "<|reserved_special_token_220|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128229": { + "content": "<|reserved_special_token_221|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128230": { + "content": "<|reserved_special_token_222|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128231": { + "content": "<|reserved_special_token_223|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128232": { + "content": "<|reserved_special_token_224|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128233": { + "content": "<|reserved_special_token_225|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128234": { + "content": "<|reserved_special_token_226|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128235": { + "content": "<|reserved_special_token_227|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128236": { + "content": "<|reserved_special_token_228|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128237": { + "content": "<|reserved_special_token_229|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128238": { + "content": "<|reserved_special_token_230|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128239": { + "content": "<|reserved_special_token_231|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128240": { + "content": "<|reserved_special_token_232|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128241": { + "content": "<|reserved_special_token_233|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128242": { + "content": "<|reserved_special_token_234|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128243": { + "content": "<|reserved_special_token_235|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128244": { + "content": "<|reserved_special_token_236|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128245": { + "content": "<|reserved_special_token_237|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128246": { + "content": "<|reserved_special_token_238|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128247": { + "content": "<|reserved_special_token_239|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128248": { + "content": "<|reserved_special_token_240|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128249": { + "content": "<|reserved_special_token_241|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128250": { + "content": "<|reserved_special_token_242|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128251": { + "content": "<|reserved_special_token_243|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128252": { + "content": "<|reserved_special_token_244|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128253": { + "content": "<|reserved_special_token_245|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128254": { + "content": "<|reserved_special_token_246|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128255": { + "content": "<|reserved_special_token_247|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + } + }, + "bos_token": "<｜begin▁of▁sentence｜>", + "clean_up_tokenization_spaces": false, + "eos_token": "<｜end▁of▁sentence｜>", + "extra_special_tokens": {}, + "legacy": true, + "model_max_length": 16384, + "pad_token": "<｜end▁of▁sentence｜>", + "sp_model_kwargs": {}, + "tokenizer_class": "LlamaTokenizerFast", + "unk_token": null, + "use_default_system_prompt": false +} diff --git a/adapter/training_args.bin b/adapter/training_args.bin new file mode 100644 index 0000000..ca8fae2 --- /dev/null +++ b/adapter/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:98a2484bdddb222de3c5a9d658c6bc6ca9f8040b634d892d790007ff9f28adbc +size 8824 diff --git a/adapter_config.json b/adapter_config.json new file mode 100644 index 0000000..2eb113c --- /dev/null +++ b/adapter_config.json @@ -0,0 +1,46 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "deepseek-ai/DeepSeek-R1-Distill-Llama-8B", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 128, + "lora_bias": false, + "lora_dropout": 0.05, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.18.0", + "qalora_group_size": 16, + "r": 64, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "down_proj", + "o_proj", + "q_proj", + "k_proj", + "v_proj", + "up_proj", + "gate_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/adapter_model.safetensors b/adapter_model.safetensors new file mode 100644 index 0000000..ba3b119 --- /dev/null +++ b/adapter_model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:60d95b10b6e140a9626a7058d5038528f2ff80148dc4569b881db56052046509 +size 40 diff --git a/all_results.json b/all_results.json new file mode 100644 index 0000000..e1f996e --- /dev/null +++ b/all_results.json @@ -0,0 +1,8 @@ +{ + "total_flos": 0.0, + "train_loss": 0.005314271912367986, + "train_runtime": 106074.3369, + "train_samples": 7000, + "train_samples_per_second": 0.226, + "train_steps_per_second": 0.005 +} \ No newline at end of file diff --git a/chat_template.jinja b/chat_template.jinja new file mode 100644 index 0000000..c2066bd --- /dev/null +++ b/chat_template.jinja @@ -0,0 +1 @@ +{% if not add_generation_prompt is defined %}{% set add_generation_prompt = false %}{% endif %}{% set ns = namespace(is_first=false, is_tool=false, is_output_first=true, system_prompt='') %}{%- for message in messages %}{%- if message['role'] == 'system' %}{% set ns.system_prompt = message['content'] %}{%- endif %}{%- endfor %}{{bos_token}}{{ns.system_prompt}}{%- for message in messages %}{%- if message['role'] == 'user' %}{%- set ns.is_tool = false -%}{{'<｜User｜>' + message['content']}}{%- endif %}{%- if message['role'] == 'assistant' and message['content'] is none %}{%- set ns.is_tool = false -%}{%- for tool in message['tool_calls']%}{%- if not ns.is_first %}{{'<｜Assistant｜><｜tool▁calls▁begin｜><｜tool▁call▁begin｜>' + tool['type'] + '<｜tool▁sep｜>' + tool['function']['name'] + '\n' + '```json' + '\n' + tool['function']['arguments'] + '\n' + '```' + '<｜tool▁call▁end｜>'}}{%- set ns.is_first = true -%}{%- else %}{{'\n' + '<｜tool▁call▁begin｜>' + tool['type'] + '<｜tool▁sep｜>' + tool['function']['name'] + '\n' + '```json' + '\n' + tool['function']['arguments'] + '\n' + '```' + '<｜tool▁call▁end｜>'}}{{'<｜tool▁calls▁end｜><｜end▁of▁sentence｜>'}}{%- endif %}{%- endfor %}{%- endif %}{%- if message['role'] == 'assistant' and message['content'] is not none %}{%- if ns.is_tool %}{{'<｜tool▁outputs▁end｜>' + message['content'] + '<｜end▁of▁sentence｜>'}}{%- set ns.is_tool = false -%}{%- else %}{% set content = message['content'] %}{% if '' in content %}{% set content = content.split('')[-1] %}{% endif %}{{'<｜Assistant｜>' + content + '<｜end▁of▁sentence｜>'}}{%- endif %}{%- endif %}{%- if message['role'] == 'tool' %}{%- set ns.is_tool = true -%}{%- if ns.is_output_first %}{{'<｜tool▁outputs▁begin｜><｜tool▁output▁begin｜>' + message['content'] + '<｜tool▁output▁end｜>'}}{%- set ns.is_output_first = false %}{%- else %}{{'\n<｜tool▁output▁begin｜>' + message['content'] + '<｜tool▁output▁end｜>'}}{%- endif %}{%- endif %}{%- endfor -%}{% if ns.is_tool %}{{'<｜tool▁outputs▁end｜>'}}{% endif %}{% if add_generation_prompt and not ns.is_tool %}{{'<｜Assistant｜>\n'}}{% endif %} \ No newline at end of file diff --git a/config.json b/config.json new file mode 100644 index 0000000..3a7db4d --- /dev/null +++ b/config.json @@ -0,0 +1,35 @@ +{ + "architectures": [ + "LlamaForCausalLM" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "bos_token_id": 128000, + "dtype": "bfloat16", + "eos_token_id": 128001, + "head_dim": 128, + "hidden_act": "silu", + "hidden_size": 4096, + "initializer_range": 0.02, + "intermediate_size": 14336, + "max_position_embeddings": 131072, + "mlp_bias": false, + "model_type": "llama", + "num_attention_heads": 32, + "num_hidden_layers": 32, + "num_key_value_heads": 8, + "pretraining_tp": 1, + "rms_norm_eps": 1e-05, + "rope_scaling": { + "factor": 8.0, + "high_freq_factor": 4.0, + "low_freq_factor": 1.0, + "original_max_position_embeddings": 8192, + "rope_type": "llama3" + }, + "rope_theta": 500000.0, + "tie_word_embeddings": false, + "transformers_version": "4.57.1", + "use_cache": true, + "vocab_size": 128256 +} diff --git a/generation_config.json b/generation_config.json new file mode 100644 index 0000000..17bba0d --- /dev/null +++ b/generation_config.json @@ -0,0 +1,9 @@ +{ + "_from_model_config": true, + "bos_token_id": 128000, + "do_sample": true, + "eos_token_id": 128001, + "temperature": 0.6, + "top_p": 0.95, + "transformers_version": "4.57.1" +} diff --git a/model-00001-of-00004.safetensors b/model-00001-of-00004.safetensors new file mode 100644 index 0000000..7741d98 --- /dev/null +++ b/model-00001-of-00004.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3f3b122712d99bf7fcadb08263d30dacfd1235dd0718a30d329f293b250993e0 +size 4976698672 diff --git a/model-00002-of-00004.safetensors b/model-00002-of-00004.safetensors new file mode 100644 index 0000000..d9e134d --- /dev/null +++ b/model-00002-of-00004.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8dbf9db854d5cfea31314aa3872566e3479e9953c417f4b79101f420737b339f +size 4999802720 diff --git a/model-00003-of-00004.safetensors b/model-00003-of-00004.safetensors new file mode 100644 index 0000000..a9a72e9 --- /dev/null +++ b/model-00003-of-00004.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ed6458e0605ca00b26b7e515c54b41d8df94d5b37672ca52aff2dcdec4844135 +size 4915916176 diff --git a/model-00004-of-00004.safetensors b/model-00004-of-00004.safetensors new file mode 100644 index 0000000..b45f92c --- /dev/null +++ b/model-00004-of-00004.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3d8773546aee65168e389a7ce275cfecf1468e48612b4e08a77f71a35760d1e2 +size 1168138808 diff --git a/model.safetensors.index.json b/model.safetensors.index.json new file mode 100644 index 0000000..5c64f1e --- /dev/null +++ b/model.safetensors.index.json @@ -0,0 +1,299 @@ +{ + "metadata": { + "total_parameters": 8030261248, + "total_size": 16060522496 + }, + "weight_map": { + "lm_head.weight": "model-00004-of-00004.safetensors", + "model.embed_tokens.weight": "model-00001-of-00004.safetensors", + "model.layers.0.input_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.0.mlp.down_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.0.mlp.up_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.1.input_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.1.mlp.down_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.1.mlp.up_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.10.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.10.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.10.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.10.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.10.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.10.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.10.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.10.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.10.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.11.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.11.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.11.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.11.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.11.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.11.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.11.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.11.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.11.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.12.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.12.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.12.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.12.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.12.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.12.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.12.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.12.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.12.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.13.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.13.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.13.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.13.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.13.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.13.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.13.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.13.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.13.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.14.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.14.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.14.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.14.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.14.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.14.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.14.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.14.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.14.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.15.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.15.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.15.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.15.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.15.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.15.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.15.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.15.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.15.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.16.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.16.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.16.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.16.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.16.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.16.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.16.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.16.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.16.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.17.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.17.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.17.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.17.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.17.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.17.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.17.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.17.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.17.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.18.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.18.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.18.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.18.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.18.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.18.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.18.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.18.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.18.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.19.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.19.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.19.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.19.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.19.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.19.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.19.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.19.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.19.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.2.input_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.2.mlp.down_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.2.mlp.up_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.20.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.20.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.20.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.20.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.20.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.20.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.20.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.20.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.20.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.21.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.21.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.21.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.21.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.21.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.21.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.21.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.21.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.21.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.22.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.22.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.22.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.22.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.22.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.22.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.22.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.22.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.22.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.23.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.23.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.23.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.23.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.23.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.23.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.23.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.23.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.23.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.24.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.24.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.24.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.24.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.24.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.24.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.24.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.24.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.24.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.25.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.25.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.25.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.25.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.25.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.25.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.25.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.25.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.25.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.26.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.26.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.26.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.26.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.26.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.26.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.26.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.26.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.26.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.27.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.27.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.27.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.27.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.27.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.27.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.27.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.27.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.27.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.28.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.28.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.28.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.28.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.28.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.28.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.28.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.28.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.28.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.29.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.29.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.29.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.29.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.29.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.29.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.29.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.29.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.29.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.3.input_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.3.mlp.down_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.3.mlp.up_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.30.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.30.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.30.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.30.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.30.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.30.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.30.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.30.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.30.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.31.input_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.31.mlp.down_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.31.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.31.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.31.post_attention_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.31.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.31.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.31.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.31.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.4.input_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.4.mlp.down_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.4.mlp.up_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.5.input_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.5.mlp.down_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.5.mlp.up_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.6.input_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.6.mlp.down_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.6.mlp.gate_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.6.mlp.up_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.7.input_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.7.mlp.down_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.7.mlp.gate_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.7.mlp.up_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.7.post_attention_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.7.self_attn.k_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.7.self_attn.o_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.7.self_attn.q_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.7.self_attn.v_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.8.input_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.8.mlp.down_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.8.mlp.gate_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.8.mlp.up_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.8.post_attention_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.8.self_attn.k_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.8.self_attn.o_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.8.self_attn.q_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.8.self_attn.v_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.9.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.9.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.9.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.9.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.9.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.9.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.9.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.9.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.9.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.norm.weight": "model-00004-of-00004.safetensors" + } +} diff --git a/special_tokens_map.json b/special_tokens_map.json new file mode 100644 index 0000000..1d385d6 --- /dev/null +++ b/special_tokens_map.json @@ -0,0 +1,23 @@ +{ + "bos_token": { + "content": "<｜begin▁of▁sentence｜>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "eos_token": { + "content": "<｜end▁of▁sentence｜>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "pad_token": { + "content": "<｜end▁of▁sentence｜>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + } +} diff --git a/tokenizer.json b/tokenizer.json new file mode 100644 index 0000000..4f47952 --- /dev/null +++ b/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d91915040cfac999d8c55f4b5bc6e67367c065e3a7a4e4b9438ce1f256addd86 +size 17209530 diff --git a/tokenizer_config.json b/tokenizer_config.json new file mode 100644 index 0000000..dd34db6 --- /dev/null +++ b/tokenizer_config.json @@ -0,0 +1,2066 @@ +{ + "add_bos_token": true, + "add_eos_token": false, + "add_prefix_space": null, + "added_tokens_decoder": { + "128000": { + "content": "<｜begin▁of▁sentence｜>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128001": { + "content": "<｜end▁of▁sentence｜>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128002": { + "content": "<|reserved_special_token_0|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128003": { + "content": "<|reserved_special_token_1|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128004": { + "content": "<|finetune_right_pad_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128005": { + "content": "<|reserved_special_token_2|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128006": { + "content": "<|start_header_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128007": { + "content": "<|end_header_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128008": { + "content": "<|eom_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128009": { + "content": "<|eot_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128010": { + "content": "<|python_tag|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128011": { + "content": "<｜User｜>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "128012": { + "content": "<｜Assistant｜>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "128013": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "128014": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "128015": { + "content": "<｜▁pad▁｜>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128016": { + "content": "<|reserved_special_token_8|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128017": { + "content": "<|reserved_special_token_9|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128018": { + "content": "<|reserved_special_token_10|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128019": { + "content": "<|reserved_special_token_11|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128020": { + "content": "<|reserved_special_token_12|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128021": { + "content": "<|reserved_special_token_13|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128022": { + "content": "<|reserved_special_token_14|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128023": { + "content": "<|reserved_special_token_15|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128024": { + "content": "<|reserved_special_token_16|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128025": { + "content": "<|reserved_special_token_17|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128026": { + "content": "<|reserved_special_token_18|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128027": { + "content": "<|reserved_special_token_19|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128028": { + "content": "<|reserved_special_token_20|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128029": { + "content": "<|reserved_special_token_21|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128030": { + "content": "<|reserved_special_token_22|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128031": { + "content": "<|reserved_special_token_23|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128032": { + "content": "<|reserved_special_token_24|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128033": { + "content": "<|reserved_special_token_25|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128034": { + "content": "<|reserved_special_token_26|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128035": { + "content": "<|reserved_special_token_27|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128036": { + "content": "<|reserved_special_token_28|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128037": { + "content": "<|reserved_special_token_29|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128038": { + "content": "<|reserved_special_token_30|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128039": { + "content": "<|reserved_special_token_31|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128040": { + "content": "<|reserved_special_token_32|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128041": { + "content": "<|reserved_special_token_33|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128042": { + "content": "<|reserved_special_token_34|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128043": { + "content": "<|reserved_special_token_35|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128044": { + "content": "<|reserved_special_token_36|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128045": { + "content": "<|reserved_special_token_37|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128046": { + "content": "<|reserved_special_token_38|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128047": { + "content": "<|reserved_special_token_39|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128048": { + "content": "<|reserved_special_token_40|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128049": { + "content": "<|reserved_special_token_41|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128050": { + "content": "<|reserved_special_token_42|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128051": { + "content": "<|reserved_special_token_43|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128052": { + "content": "<|reserved_special_token_44|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128053": { + "content": "<|reserved_special_token_45|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128054": { + "content": "<|reserved_special_token_46|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128055": { + "content": "<|reserved_special_token_47|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128056": { + "content": "<|reserved_special_token_48|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128057": { + "content": "<|reserved_special_token_49|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128058": { + "content": "<|reserved_special_token_50|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128059": { + "content": "<|reserved_special_token_51|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128060": { + "content": "<|reserved_special_token_52|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128061": { + "content": "<|reserved_special_token_53|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128062": { + "content": "<|reserved_special_token_54|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128063": { + "content": "<|reserved_special_token_55|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128064": { + "content": "<|reserved_special_token_56|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128065": { + "content": "<|reserved_special_token_57|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128066": { + "content": "<|reserved_special_token_58|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128067": { + "content": "<|reserved_special_token_59|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128068": { + "content": "<|reserved_special_token_60|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128069": { + "content": "<|reserved_special_token_61|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128070": { + "content": "<|reserved_special_token_62|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128071": { + "content": "<|reserved_special_token_63|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128072": { + "content": "<|reserved_special_token_64|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128073": { + "content": "<|reserved_special_token_65|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128074": { + "content": "<|reserved_special_token_66|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128075": { + "content": "<|reserved_special_token_67|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128076": { + "content": "<|reserved_special_token_68|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128077": { + "content": "<|reserved_special_token_69|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128078": { + "content": "<|reserved_special_token_70|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128079": { + "content": "<|reserved_special_token_71|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128080": { + "content": "<|reserved_special_token_72|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128081": { + "content": "<|reserved_special_token_73|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128082": { + "content": "<|reserved_special_token_74|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128083": { + "content": "<|reserved_special_token_75|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128084": { + "content": "<|reserved_special_token_76|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128085": { + "content": "<|reserved_special_token_77|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128086": { + "content": "<|reserved_special_token_78|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128087": { + "content": "<|reserved_special_token_79|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128088": { + "content": "<|reserved_special_token_80|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128089": { + "content": "<|reserved_special_token_81|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128090": { + "content": "<|reserved_special_token_82|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128091": { + "content": "<|reserved_special_token_83|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128092": { + "content": "<|reserved_special_token_84|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128093": { + "content": "<|reserved_special_token_85|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128094": { + "content": "<|reserved_special_token_86|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128095": { + "content": "<|reserved_special_token_87|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128096": { + "content": "<|reserved_special_token_88|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128097": { + "content": "<|reserved_special_token_89|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128098": { + "content": "<|reserved_special_token_90|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128099": { + "content": "<|reserved_special_token_91|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128100": { + "content": "<|reserved_special_token_92|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128101": { + "content": "<|reserved_special_token_93|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128102": { + "content": "<|reserved_special_token_94|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128103": { + "content": "<|reserved_special_token_95|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128104": { + "content": "<|reserved_special_token_96|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128105": { + "content": "<|reserved_special_token_97|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128106": { + "content": "<|reserved_special_token_98|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128107": { + "content": "<|reserved_special_token_99|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128108": { + "content": "<|reserved_special_token_100|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128109": { + "content": "<|reserved_special_token_101|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128110": { + "content": "<|reserved_special_token_102|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128111": { + "content": "<|reserved_special_token_103|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128112": { + "content": "<|reserved_special_token_104|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128113": { + "content": "<|reserved_special_token_105|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128114": { + "content": "<|reserved_special_token_106|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128115": { + "content": "<|reserved_special_token_107|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128116": { + "content": "<|reserved_special_token_108|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128117": { + "content": "<|reserved_special_token_109|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128118": { + "content": "<|reserved_special_token_110|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128119": { + "content": "<|reserved_special_token_111|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128120": { + "content": "<|reserved_special_token_112|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128121": { + "content": "<|reserved_special_token_113|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128122": { + "content": "<|reserved_special_token_114|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128123": { + "content": "<|reserved_special_token_115|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128124": { + "content": "<|reserved_special_token_116|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128125": { + "content": "<|reserved_special_token_117|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128126": { + "content": "<|reserved_special_token_118|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128127": { + "content": "<|reserved_special_token_119|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128128": { + "content": "<|reserved_special_token_120|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128129": { + "content": "<|reserved_special_token_121|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128130": { + "content": "<|reserved_special_token_122|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128131": { + "content": "<|reserved_special_token_123|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128132": { + "content": "<|reserved_special_token_124|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128133": { + "content": "<|reserved_special_token_125|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128134": { + "content": "<|reserved_special_token_126|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128135": { + "content": "<|reserved_special_token_127|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128136": { + "content": "<|reserved_special_token_128|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128137": { + "content": "<|reserved_special_token_129|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128138": { + "content": "<|reserved_special_token_130|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128139": { + "content": "<|reserved_special_token_131|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128140": { + "content": "<|reserved_special_token_132|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128141": { + "content": "<|reserved_special_token_133|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128142": { + "content": "<|reserved_special_token_134|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128143": { + "content": "<|reserved_special_token_135|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128144": { + "content": "<|reserved_special_token_136|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128145": { + "content": "<|reserved_special_token_137|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128146": { + "content": "<|reserved_special_token_138|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128147": { + "content": "<|reserved_special_token_139|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128148": { + "content": "<|reserved_special_token_140|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128149": { + "content": "<|reserved_special_token_141|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128150": { + "content": "<|reserved_special_token_142|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128151": { + "content": "<|reserved_special_token_143|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128152": { + "content": "<|reserved_special_token_144|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128153": { + "content": "<|reserved_special_token_145|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128154": { + "content": "<|reserved_special_token_146|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128155": { + "content": "<|reserved_special_token_147|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128156": { + "content": "<|reserved_special_token_148|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128157": { + "content": "<|reserved_special_token_149|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128158": { + "content": "<|reserved_special_token_150|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128159": { + "content": "<|reserved_special_token_151|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128160": { + "content": "<|reserved_special_token_152|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128161": { + "content": "<|reserved_special_token_153|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128162": { + "content": "<|reserved_special_token_154|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128163": { + "content": "<|reserved_special_token_155|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128164": { + "content": "<|reserved_special_token_156|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128165": { + "content": "<|reserved_special_token_157|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128166": { + "content": "<|reserved_special_token_158|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128167": { + "content": "<|reserved_special_token_159|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128168": { + "content": "<|reserved_special_token_160|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128169": { + "content": "<|reserved_special_token_161|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128170": { + "content": "<|reserved_special_token_162|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128171": { + "content": "<|reserved_special_token_163|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128172": { + "content": "<|reserved_special_token_164|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128173": { + "content": "<|reserved_special_token_165|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128174": { + "content": "<|reserved_special_token_166|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128175": { + "content": "<|reserved_special_token_167|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128176": { + "content": "<|reserved_special_token_168|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128177": { + "content": "<|reserved_special_token_169|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128178": { + "content": "<|reserved_special_token_170|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128179": { + "content": "<|reserved_special_token_171|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128180": { + "content": "<|reserved_special_token_172|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128181": { + "content": "<|reserved_special_token_173|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128182": { + "content": "<|reserved_special_token_174|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128183": { + "content": "<|reserved_special_token_175|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128184": { + "content": "<|reserved_special_token_176|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128185": { + "content": "<|reserved_special_token_177|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128186": { + "content": "<|reserved_special_token_178|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128187": { + "content": "<|reserved_special_token_179|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128188": { + "content": "<|reserved_special_token_180|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128189": { + "content": "<|reserved_special_token_181|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128190": { + "content": "<|reserved_special_token_182|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128191": { + "content": "<|reserved_special_token_183|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128192": { + "content": "<|reserved_special_token_184|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128193": { + "content": "<|reserved_special_token_185|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128194": { + "content": "<|reserved_special_token_186|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128195": { + "content": "<|reserved_special_token_187|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128196": { + "content": "<|reserved_special_token_188|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128197": { + "content": "<|reserved_special_token_189|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128198": { + "content": "<|reserved_special_token_190|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128199": { + "content": "<|reserved_special_token_191|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128200": { + "content": "<|reserved_special_token_192|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128201": { + "content": "<|reserved_special_token_193|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128202": { + "content": "<|reserved_special_token_194|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128203": { + "content": "<|reserved_special_token_195|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128204": { + "content": "<|reserved_special_token_196|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128205": { + "content": "<|reserved_special_token_197|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128206": { + "content": "<|reserved_special_token_198|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128207": { + "content": "<|reserved_special_token_199|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128208": { + "content": "<|reserved_special_token_200|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128209": { + "content": "<|reserved_special_token_201|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128210": { + "content": "<|reserved_special_token_202|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128211": { + "content": "<|reserved_special_token_203|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128212": { + "content": "<|reserved_special_token_204|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128213": { + "content": "<|reserved_special_token_205|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128214": { + "content": "<|reserved_special_token_206|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128215": { + "content": "<|reserved_special_token_207|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128216": { + "content": "<|reserved_special_token_208|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128217": { + "content": "<|reserved_special_token_209|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128218": { + "content": "<|reserved_special_token_210|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128219": { + "content": "<|reserved_special_token_211|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128220": { + "content": "<|reserved_special_token_212|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128221": { + "content": "<|reserved_special_token_213|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128222": { + "content": "<|reserved_special_token_214|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128223": { + "content": "<|reserved_special_token_215|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128224": { + "content": "<|reserved_special_token_216|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128225": { + "content": "<|reserved_special_token_217|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128226": { + "content": "<|reserved_special_token_218|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128227": { + "content": "<|reserved_special_token_219|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128228": { + "content": "<|reserved_special_token_220|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128229": { + "content": "<|reserved_special_token_221|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128230": { + "content": "<|reserved_special_token_222|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128231": { + "content": "<|reserved_special_token_223|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128232": { + "content": "<|reserved_special_token_224|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128233": { + "content": "<|reserved_special_token_225|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128234": { + "content": "<|reserved_special_token_226|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128235": { + "content": "<|reserved_special_token_227|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128236": { + "content": "<|reserved_special_token_228|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128237": { + "content": "<|reserved_special_token_229|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128238": { + "content": "<|reserved_special_token_230|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128239": { + "content": "<|reserved_special_token_231|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128240": { + "content": "<|reserved_special_token_232|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128241": { + "content": "<|reserved_special_token_233|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128242": { + "content": "<|reserved_special_token_234|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128243": { + "content": "<|reserved_special_token_235|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128244": { + "content": "<|reserved_special_token_236|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128245": { + "content": "<|reserved_special_token_237|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128246": { + "content": "<|reserved_special_token_238|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128247": { + "content": "<|reserved_special_token_239|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128248": { + "content": "<|reserved_special_token_240|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128249": { + "content": "<|reserved_special_token_241|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128250": { + "content": "<|reserved_special_token_242|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128251": { + "content": "<|reserved_special_token_243|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128252": { + "content": "<|reserved_special_token_244|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128253": { + "content": "<|reserved_special_token_245|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128254": { + "content": "<|reserved_special_token_246|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128255": { + "content": "<|reserved_special_token_247|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + } + }, + "bos_token": "<｜begin▁of▁sentence｜>", + "clean_up_tokenization_spaces": false, + "eos_token": "<｜end▁of▁sentence｜>", + "extra_special_tokens": {}, + "legacy": true, + "model_max_length": 16384, + "pad_token": "<｜end▁of▁sentence｜>", + "sp_model_kwargs": {}, + "tokenizer_class": "LlamaTokenizerFast", + "unk_token": null, + "use_default_system_prompt": false +} diff --git a/train_results.json b/train_results.json new file mode 100644 index 0000000..e1f996e --- /dev/null +++ b/train_results.json @@ -0,0 +1,8 @@ +{ + "total_flos": 0.0, + "train_loss": 0.005314271912367986, + "train_runtime": 106074.3369, + "train_samples": 7000, + "train_samples_per_second": 0.226, + "train_steps_per_second": 0.005 +} \ No newline at end of file diff --git a/trainer_state.json b/trainer_state.json new file mode 100644 index 0000000..a4a9964 --- /dev/null +++ b/trainer_state.json @@ -0,0 +1,11543 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 0.5714285714285714, + "eval_steps": 500, + "global_step": 500, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "clip_fraction": 0.0, + "completion_length": 2523.270866394043, + "epoch": 0.001142857142857143, + "grad_norm": 0.06654668599367142, + "kl": 0.0, + "lambda_div_used": 0.6164634302258492, + "learning_rate": 0.0, + "loss": -0.0258, + "reward": -0.14551701629534364, + "reward_after_mean": -0.14551701629534364, + "reward_after_std": 0.6225011153146625, + "reward_before_mean": 0.17862090840935707, + "reward_before_std": 0.5394803490489721, + "reward_change_max": 0.0, + "reward_change_mean": -0.3241379093378782, + "reward_change_min": -0.5330121107399464, + "reward_change_std": 0.19750467501580715, + "reward_std": 0.622501116245985, + "rewards/accuracy_reward": 0.2500000037252903, + "rewards/cosine_scaled_reward": -0.0713790925219655, + "step": 1 + }, + { + "clip_fraction": 0.0, + "completion_length": 2684.583366394043, + "epoch": 0.002285714285714286, + "grad_norm": 0.07638121396303177, + "kl": 0.0, + "lambda_div_used": 0.5910675376653671, + "learning_rate": 5e-08, + "loss": 0.001, + "reward": -0.07136534340679646, + "reward_after_mean": -0.07136534340679646, + "reward_after_std": 0.5026816055178642, + "reward_before_mean": 0.33918463438749313, + "reward_before_std": 0.41114553064107895, + "reward_change_max": 0.0, + "reward_change_mean": -0.4105500001460314, + "reward_change_min": -0.619081187993288, + "reward_change_std": 0.23483010660856962, + "reward_std": 0.5026816166937351, + "rewards/accuracy_reward": 0.29166667349636555, + "rewards/cosine_scaled_reward": 0.047517990693449974, + "step": 2 + }, + { + "clip_fraction": 0.0, + "completion_length": 2966.9166870117188, + "epoch": 0.0034285714285714284, + "grad_norm": 0.07826078683137894, + "kl": 5.805492401123047e-05, + "lambda_div_used": 0.5747007578611374, + "learning_rate": 1e-07, + "loss": 0.0118, + "reward": -0.3506452329456806, + "reward_after_mean": -0.3506452329456806, + "reward_after_std": 0.40795029513537884, + "reward_before_mean": -0.06986081041395664, + "reward_before_std": 0.3391748256981373, + "reward_change_max": 0.0, + "reward_change_mean": -0.280784422531724, + "reward_change_min": -0.45085202157497406, + "reward_change_std": 0.16326917707920074, + "reward_std": 0.4079503044486046, + "rewards/accuracy_reward": 0.06250000186264515, + "rewards/cosine_scaled_reward": -0.1323608043603599, + "step": 3 + }, + { + "clip_fraction": 0.0, + "completion_length": 1377.000015258789, + "epoch": 0.004571428571428572, + "grad_norm": 0.10993114858865738, + "kl": 3.951042890548706e-05, + "lambda_div_used": 0.5788158848881721, + "learning_rate": 1.5e-07, + "loss": 0.0214, + "reward": -0.24558139964938164, + "reward_after_mean": -0.24558139964938164, + "reward_after_std": 0.3998244144022465, + "reward_before_mean": 0.06915237568318844, + "reward_before_std": 0.36195528600364923, + "reward_change_max": 0.0, + "reward_change_mean": -0.3147337753325701, + "reward_change_min": -0.4869973622262478, + "reward_change_std": 0.1927295122295618, + "reward_std": 0.3998244162648916, + "rewards/accuracy_reward": 0.1666666716337204, + "rewards/cosine_scaled_reward": -0.09751429967582226, + "step": 4 + }, + { + "clip_fraction": 0.0, + "completion_length": 3310.4583587646484, + "epoch": 0.005714285714285714, + "grad_norm": 0.06377790123224258, + "kl": 5.4255127906799316e-05, + "lambda_div_used": 0.61885916441679, + "learning_rate": 2e-07, + "loss": 0.0428, + "reward": -0.25358792673796415, + "reward_after_mean": -0.25358792673796415, + "reward_after_std": 0.5858059301972389, + "reward_before_mean": 0.0003254720941185951, + "reward_before_std": 0.5446656532585621, + "reward_change_max": 0.0, + "reward_change_mean": -0.25391339510679245, + "reward_change_min": -0.45586229115724564, + "reward_change_std": 0.16313681472092867, + "reward_std": 0.5858059376478195, + "rewards/accuracy_reward": 0.1250000037252903, + "rewards/cosine_scaled_reward": -0.12467453628778458, + "step": 5 + }, + { + "clip_fraction": 0.0, + "completion_length": 2887.8541717529297, + "epoch": 0.006857142857142857, + "grad_norm": 0.06263996660709381, + "kl": 5.511939525604248e-05, + "lambda_div_used": 0.5606320649385452, + "learning_rate": 2.5e-07, + "loss": 0.0618, + "reward": -0.4456054698675871, + "reward_after_mean": -0.4456054698675871, + "reward_after_std": 0.3509902711957693, + "reward_before_mean": -0.1859958479180932, + "reward_before_std": 0.2702419590204954, + "reward_change_max": 0.0, + "reward_change_mean": -0.2596096061170101, + "reward_change_min": -0.38109203428030014, + "reward_change_std": 0.14125030301511288, + "reward_std": 0.3509902749210596, + "rewards/accuracy_reward": 0.0416666679084301, + "rewards/cosine_scaled_reward": -0.2276625158265233, + "step": 6 + }, + { + "clip_fraction": 0.0, + "completion_length": 2572.3542098999023, + "epoch": 0.008, + "grad_norm": 0.06695970892906189, + "kl": 4.0218234062194824e-05, + "lambda_div_used": 0.6162339821457863, + "learning_rate": 3e-07, + "loss": 0.0453, + "reward": -0.10290078446269035, + "reward_after_mean": -0.10290078446269035, + "reward_after_std": 0.566169198602438, + "reward_before_mean": 0.22045390354469419, + "reward_before_std": 0.5333282891660929, + "reward_change_max": 0.0, + "reward_change_mean": -0.32335469499230385, + "reward_change_min": -0.5540991388261318, + "reward_change_std": 0.20453903079032898, + "reward_std": 0.566169211640954, + "rewards/accuracy_reward": 0.20833334140479565, + "rewards/cosine_scaled_reward": 0.012120556086301804, + "step": 7 + }, + { + "clip_fraction": 0.0, + "completion_length": 2173.7500228881836, + "epoch": 0.009142857142857144, + "grad_norm": 0.07779484242200851, + "kl": 3.185681998729706e-05, + "lambda_div_used": 0.63105358928442, + "learning_rate": 3.5e-07, + "loss": 0.0071, + "reward": 0.09723488846793771, + "reward_after_mean": 0.09723488846793771, + "reward_after_std": 0.6893859151750803, + "reward_before_mean": 0.5079077246482484, + "reward_before_std": 0.6077435212209821, + "reward_change_max": 0.0, + "reward_change_mean": -0.410672839730978, + "reward_change_min": -0.638349361717701, + "reward_change_std": 0.2494220733642578, + "reward_std": 0.6893859468400478, + "rewards/accuracy_reward": 0.3750000074505806, + "rewards/cosine_scaled_reward": 0.1329077403061092, + "step": 8 + }, + { + "clip_fraction": 0.0, + "completion_length": 2828.229179382324, + "epoch": 0.010285714285714285, + "grad_norm": 0.09342571347951889, + "kl": 4.9620866775512695e-05, + "lambda_div_used": 0.6150719821453094, + "learning_rate": 4e-07, + "loss": -0.0132, + "reward": -0.18431122601032257, + "reward_after_mean": -0.18431122601032257, + "reward_after_std": 0.5804070886224508, + "reward_before_mean": 0.09753156686201692, + "reward_before_std": 0.5302032623440027, + "reward_change_max": 0.0, + "reward_change_mean": -0.2818427961319685, + "reward_change_min": -0.46669338643550873, + "reward_change_std": 0.18001185078173876, + "reward_std": 0.5804071053862572, + "rewards/accuracy_reward": 0.1666666679084301, + "rewards/cosine_scaled_reward": -0.0691350968554616, + "step": 9 + }, + { + "clip_fraction": 0.0, + "completion_length": 2469.7083587646484, + "epoch": 0.011428571428571429, + "grad_norm": 0.09024360775947571, + "kl": 3.8331374526023865e-05, + "lambda_div_used": 0.63355503231287, + "learning_rate": 4.5e-07, + "loss": -0.001, + "reward": -0.04430920258164406, + "reward_after_mean": -0.04430920258164406, + "reward_after_std": 0.6170141901820898, + "reward_before_mean": 0.2625937759876251, + "reward_before_std": 0.6259458847343922, + "reward_change_max": 0.0, + "reward_change_mean": -0.3069029748439789, + "reward_change_min": -0.5734602250158787, + "reward_change_std": 0.22167105227708817, + "reward_std": 0.6170141994953156, + "rewards/accuracy_reward": 0.2500000074505806, + "rewards/cosine_scaled_reward": 0.012593759223818779, + "step": 10 + }, + { + "clip_fraction": 0.0, + "completion_length": 3367.500015258789, + "epoch": 0.012571428571428572, + "grad_norm": 0.05533275753259659, + "kl": 5.245208740234375e-05, + "lambda_div_used": 0.5749180987477303, + "learning_rate": 5e-07, + "loss": -0.0467, + "reward": -0.35878527723252773, + "reward_after_mean": -0.35878527723252773, + "reward_after_std": 0.4030665699392557, + "reward_before_mean": -0.09556051343679428, + "reward_before_std": 0.3429161449894309, + "reward_change_max": 0.0, + "reward_change_mean": -0.2632247470319271, + "reward_change_min": -0.4096112735569477, + "reward_change_std": 0.1540710162371397, + "reward_std": 0.40306657925248146, + "rewards/accuracy_reward": 0.10416666977107525, + "rewards/cosine_scaled_reward": -0.19972719065845013, + "step": 11 + }, + { + "clip_fraction": 0.0, + "completion_length": 2016.958351135254, + "epoch": 0.013714285714285714, + "grad_norm": 0.08992662280797958, + "kl": 3.9637088775634766e-05, + "lambda_div_used": 0.6247389540076256, + "learning_rate": 5.5e-07, + "loss": 0.0429, + "reward": -0.10162727534770966, + "reward_after_mean": -0.10162727534770966, + "reward_after_std": 0.594907833263278, + "reward_before_mean": 0.19102710485458374, + "reward_before_std": 0.5805315412580967, + "reward_change_max": 0.0, + "reward_change_mean": -0.292654387652874, + "reward_change_min": -0.5117304362356663, + "reward_change_std": 0.20085694547742605, + "reward_std": 0.5949078388512135, + "rewards/accuracy_reward": 0.22916666977107525, + "rewards/cosine_scaled_reward": -0.03813957702368498, + "step": 12 + }, + { + "clip_fraction": 0.0, + "completion_length": 2905.0833587646484, + "epoch": 0.014857142857142857, + "grad_norm": 0.0641385093331337, + "kl": 4.194676876068115e-05, + "lambda_div_used": 0.5766140297055244, + "learning_rate": 6e-07, + "loss": 0.0437, + "reward": -0.3046809285879135, + "reward_after_mean": -0.3046809285879135, + "reward_after_std": 0.399059085175395, + "reward_before_mean": -0.018187658861279488, + "reward_before_std": 0.35123275220394135, + "reward_change_max": 0.0, + "reward_change_mean": -0.28649328649044037, + "reward_change_min": -0.4807186797261238, + "reward_change_std": 0.17643271200358868, + "reward_std": 0.399059085175395, + "rewards/accuracy_reward": 0.1250000037252903, + "rewards/cosine_scaled_reward": -0.1431876514106989, + "step": 13 + }, + { + "clip_fraction": 0.0, + "completion_length": 2520.312530517578, + "epoch": 0.016, + "grad_norm": 0.07475250959396362, + "kl": 3.432855010032654e-05, + "lambda_div_used": 0.5991310104727745, + "learning_rate": 6.5e-07, + "loss": -0.0332, + "reward": -0.22936711832880974, + "reward_after_mean": -0.22936711832880974, + "reward_after_std": 0.5019008349627256, + "reward_before_mean": 0.04907496925443411, + "reward_before_std": 0.4585955021902919, + "reward_change_max": 0.0, + "reward_change_mean": -0.27844210527837276, + "reward_change_min": -0.4685846194624901, + "reward_change_std": 0.17394172679632902, + "reward_std": 0.5019008629024029, + "rewards/accuracy_reward": 0.14583333767950535, + "rewards/cosine_scaled_reward": -0.09675835724920034, + "step": 14 + }, + { + "clip_fraction": 0.0, + "completion_length": 2717.0416984558105, + "epoch": 0.017142857142857144, + "grad_norm": 0.08986662328243256, + "kl": 4.3585896492004395e-05, + "lambda_div_used": 0.5765259638428688, + "learning_rate": 7e-07, + "loss": 0.0355, + "reward": -0.16787780448794365, + "reward_after_mean": -0.16787780448794365, + "reward_after_std": 0.4601633083075285, + "reward_before_mean": 0.22437161579728127, + "reward_before_std": 0.3496173685416579, + "reward_change_max": 0.0, + "reward_change_mean": -0.3922494389116764, + "reward_change_min": -0.5602193549275398, + "reward_change_std": 0.2197399353608489, + "reward_std": 0.46016331762075424, + "rewards/accuracy_reward": 0.20833333395421505, + "rewards/cosine_scaled_reward": 0.016038289293646812, + "step": 15 + }, + { + "clip_fraction": 0.0, + "completion_length": 3395.1875, + "epoch": 0.018285714285714287, + "grad_norm": 0.05402011796832085, + "kl": 4.659593105316162e-05, + "lambda_div_used": 0.5638558194041252, + "learning_rate": 7.5e-07, + "loss": 0.032, + "reward": -0.432980858720839, + "reward_after_mean": -0.432980858720839, + "reward_after_std": 0.3415633924305439, + "reward_before_mean": -0.16444075386971235, + "reward_before_std": 0.2898183651268482, + "reward_change_max": 0.0, + "reward_change_mean": -0.26854010485112667, + "reward_change_min": -0.45487307757139206, + "reward_change_std": 0.16323526203632355, + "reward_std": 0.34156340546905994, + "rewards/accuracy_reward": 0.0416666679084301, + "rewards/cosine_scaled_reward": -0.20610742270946503, + "step": 16 + }, + { + "clip_fraction": 0.0, + "completion_length": 2164.020854949951, + "epoch": 0.019428571428571427, + "grad_norm": 0.1185457855463028, + "kl": 4.2147934436798096e-05, + "lambda_div_used": 0.5763917565345764, + "learning_rate": 8e-07, + "loss": 0.0302, + "reward": -0.2335935328155756, + "reward_after_mean": -0.2335935328155756, + "reward_after_std": 0.4716028142720461, + "reward_before_mean": 0.13111361488699913, + "reward_before_std": 0.3423871146515012, + "reward_change_max": 0.0, + "reward_change_mean": -0.36470715142786503, + "reward_change_min": -0.5119686089456081, + "reward_change_std": 0.19370271731168032, + "reward_std": 0.471602825447917, + "rewards/accuracy_reward": 0.18750000186264515, + "rewards/cosine_scaled_reward": -0.05638639978133142, + "step": 17 + }, + { + "clip_fraction": 0.0, + "completion_length": 3079.4583740234375, + "epoch": 0.02057142857142857, + "grad_norm": 0.05137661099433899, + "kl": 3.637373447418213e-05, + "lambda_div_used": 0.6073044687509537, + "learning_rate": 8.499999999999999e-07, + "loss": 0.0478, + "reward": -0.057250723242759705, + "reward_after_mean": -0.057250723242759705, + "reward_after_std": 0.5715843215584755, + "reward_before_mean": 0.3262214660644531, + "reward_before_std": 0.5018989769741893, + "reward_change_max": 0.0, + "reward_change_mean": -0.38347217813134193, + "reward_change_min": -0.6260374560952187, + "reward_change_std": 0.24400013033300638, + "reward_std": 0.571584340184927, + "rewards/accuracy_reward": 0.2916666679084301, + "rewards/cosine_scaled_reward": 0.034554785932414234, + "step": 18 + }, + { + "clip_fraction": 0.0, + "completion_length": 2889.8542289733887, + "epoch": 0.021714285714285714, + "grad_norm": 0.07495337724685669, + "kl": 4.096329212188721e-05, + "lambda_div_used": 0.6219106838107109, + "learning_rate": 9e-07, + "loss": 0.036, + "reward": 0.1660416293889284, + "reward_after_mean": 0.1660416293889284, + "reward_after_std": 0.6783627849072218, + "reward_before_mean": 0.6544073540717363, + "reward_before_std": 0.5640754774212837, + "reward_change_max": 0.0, + "reward_change_mean": -0.488365713506937, + "reward_change_min": -0.758969146758318, + "reward_change_std": 0.2900321548804641, + "reward_std": 0.6783628333359957, + "rewards/accuracy_reward": 0.45833334140479565, + "rewards/cosine_scaled_reward": 0.19607400865061209, + "step": 19 + }, + { + "clip_fraction": 0.0, + "completion_length": 2114.8333587646484, + "epoch": 0.022857142857142857, + "grad_norm": 0.0912039652466774, + "kl": 2.7514994144439697e-05, + "lambda_div_used": 0.6171177923679352, + "learning_rate": 9.499999999999999e-07, + "loss": -0.0338, + "reward": -0.12753646727651358, + "reward_after_mean": -0.12753646727651358, + "reward_after_std": 0.6479649767279625, + "reward_before_mean": 0.2073977841064334, + "reward_before_std": 0.5446468549780548, + "reward_change_max": 0.0, + "reward_change_mean": -0.33493425138294697, + "reward_change_min": -0.5443090125918388, + "reward_change_std": 0.20240973494946957, + "reward_std": 0.6479649972170591, + "rewards/accuracy_reward": 0.2500000037252903, + "rewards/cosine_scaled_reward": -0.0426022089086473, + "step": 20 + }, + { + "clip_fraction": 0.0, + "completion_length": 2578.3125, + "epoch": 0.024, + "grad_norm": 0.1021808609366417, + "kl": 4.0903687477111816e-05, + "lambda_div_used": 0.5696183741092682, + "learning_rate": 1e-06, + "loss": 0.0243, + "reward": -0.16086186096072197, + "reward_after_mean": -0.16086186096072197, + "reward_after_std": 0.4245347697287798, + "reward_before_mean": 0.23232585442019626, + "reward_before_std": 0.3138282438740134, + "reward_change_max": 0.0, + "reward_change_mean": -0.3931876849383116, + "reward_change_min": -0.5524330623447895, + "reward_change_std": 0.2125744568184018, + "reward_std": 0.424534784629941, + "rewards/accuracy_reward": 0.25000000558793545, + "rewards/cosine_scaled_reward": -0.01767415925860405, + "step": 21 + }, + { + "clip_fraction": 0.0, + "completion_length": 1644.437572479248, + "epoch": 0.025142857142857144, + "grad_norm": 0.11418648809194565, + "kl": 2.1685846149921417e-05, + "lambda_div_used": 0.5976943448185921, + "learning_rate": 9.99931462820376e-07, + "loss": -0.0304, + "reward": -0.10828415304422379, + "reward_after_mean": -0.10828415304422379, + "reward_after_std": 0.5476059578359127, + "reward_before_mean": 0.2575272931717336, + "reward_before_std": 0.4531834872905165, + "reward_change_max": 0.0, + "reward_change_mean": -0.36581144109368324, + "reward_change_min": -0.5498910807073116, + "reward_change_std": 0.21405170671641827, + "reward_std": 0.5476059839129448, + "rewards/accuracy_reward": 0.29166667349636555, + "rewards/cosine_scaled_reward": -0.03413938358426094, + "step": 22 + }, + { + "clip_fraction": 0.0, + "completion_length": 2188.687530517578, + "epoch": 0.026285714285714287, + "grad_norm": 0.0898994579911232, + "kl": 3.7088990211486816e-05, + "lambda_div_used": 0.6402537003159523, + "learning_rate": 9.997258721585931e-07, + "loss": -0.0037, + "reward": -0.13877022732049227, + "reward_after_mean": -0.13877022732049227, + "reward_after_std": 0.6969181355088949, + "reward_before_mean": 0.11387635703431442, + "reward_before_std": 0.6502415342256427, + "reward_change_max": 0.0, + "reward_change_mean": -0.2526465691626072, + "reward_change_min": -0.45175985619425774, + "reward_change_std": 0.1627071350812912, + "reward_std": 0.6969181522727013, + "rewards/accuracy_reward": 0.16666666977107525, + "rewards/cosine_scaled_reward": -0.05279031861573458, + "step": 23 + }, + { + "clip_fraction": 0.0, + "completion_length": 2394.3958740234375, + "epoch": 0.027428571428571427, + "grad_norm": 0.09361977875232697, + "kl": 2.0720064640045166e-05, + "lambda_div_used": 0.6309552267193794, + "learning_rate": 9.993832906395582e-07, + "loss": 0.0666, + "reward": 0.0798279655573424, + "reward_after_mean": 0.0798279655573424, + "reward_after_std": 0.6638341955840588, + "reward_before_mean": 0.4744788520038128, + "reward_before_std": 0.6099728401750326, + "reward_change_max": 0.0, + "reward_change_mean": -0.3946509025990963, + "reward_change_min": -0.6427228525280952, + "reward_change_std": 0.25018193013966084, + "reward_std": 0.6638342067599297, + "rewards/accuracy_reward": 0.35416667349636555, + "rewards/cosine_scaled_reward": 0.12031219294294715, + "step": 24 + }, + { + "clip_fraction": 0.0, + "completion_length": 2358.583366394043, + "epoch": 0.02857142857142857, + "grad_norm": 0.07725252956151962, + "kl": 4.092603921890259e-05, + "lambda_div_used": 0.6378427669405937, + "learning_rate": 9.989038226169207e-07, + "loss": -0.0557, + "reward": -0.15363326482474804, + "reward_after_mean": -0.15363326482474804, + "reward_after_std": 0.6477887704968452, + "reward_before_mean": 0.10259643197059631, + "reward_before_std": 0.6456613540649414, + "reward_change_max": 0.0, + "reward_change_mean": -0.25622970052063465, + "reward_change_min": -0.5558900497853756, + "reward_change_std": 0.19845533184707165, + "reward_std": 0.6477888077497482, + "rewards/accuracy_reward": 0.1875000037252903, + "rewards/cosine_scaled_reward": -0.08490357082337141, + "step": 25 + }, + { + "clip_fraction": 0.0, + "completion_length": 2850.937530517578, + "epoch": 0.029714285714285714, + "grad_norm": 0.06806578487157822, + "kl": 3.5643577575683594e-05, + "lambda_div_used": 0.5781113430857658, + "learning_rate": 9.982876141412855e-07, + "loss": 0.005, + "reward": -0.25517464708536863, + "reward_after_mean": -0.25517464708536863, + "reward_after_std": 0.47686139307916164, + "reward_before_mean": 0.09097394905984402, + "reward_before_std": 0.3564971052110195, + "reward_change_max": 0.0, + "reward_change_mean": -0.3461485952138901, + "reward_change_min": -0.4805891402065754, + "reward_change_std": 0.18107541371136904, + "reward_std": 0.476861409842968, + "rewards/accuracy_reward": 0.1666666679084301, + "rewards/cosine_scaled_reward": -0.07569271977990866, + "step": 26 + }, + { + "clip_fraction": 0.0, + "completion_length": 2795.3750534057617, + "epoch": 0.030857142857142857, + "grad_norm": 0.0670829713344574, + "kl": 5.0067901611328125e-05, + "lambda_div_used": 0.5909813195466995, + "learning_rate": 9.975348529157229e-07, + "loss": 0.038, + "reward": -0.30352935567498207, + "reward_after_mean": -0.30352935567498207, + "reward_after_std": 0.4782011900097132, + "reward_before_mean": -0.04106062464416027, + "reward_before_std": 0.42168071679770947, + "reward_change_max": 0.0, + "reward_change_mean": -0.2624687273055315, + "reward_change_min": -0.441840048879385, + "reward_change_std": 0.16586182732135057, + "reward_std": 0.47820119373500347, + "rewards/accuracy_reward": 0.12500000186264515, + "rewards/cosine_scaled_reward": -0.16606062231585383, + "step": 27 + }, + { + "clip_fraction": 0.0, + "completion_length": 2662.562530517578, + "epoch": 0.032, + "grad_norm": 0.07541470974683762, + "kl": 4.6879053115844727e-05, + "lambda_div_used": 0.6455363035202026, + "learning_rate": 9.96645768238595e-07, + "loss": -0.0548, + "reward": -0.01765006221830845, + "reward_after_mean": -0.01765006221830845, + "reward_after_std": 0.7017391249537468, + "reward_before_mean": 0.26933106034994125, + "reward_before_std": 0.6768321208655834, + "reward_change_max": 0.0, + "reward_change_mean": -0.2869811188429594, + "reward_change_min": -0.5125514604151249, + "reward_change_std": 0.19719962775707245, + "reward_std": 0.7017391547560692, + "rewards/accuracy_reward": 0.29166667349636555, + "rewards/cosine_scaled_reward": -0.022335614077746868, + "step": 28 + }, + { + "clip_fraction": 0.0, + "completion_length": 2988.1041717529297, + "epoch": 0.03314285714285714, + "grad_norm": 0.07968341559171677, + "kl": 4.7832727432250977e-05, + "lambda_div_used": 0.5851128473877907, + "learning_rate": 9.956206309337066e-07, + "loss": -0.1054, + "reward": -0.38509707152843475, + "reward_after_mean": -0.38509707152843475, + "reward_after_std": 0.4501145612448454, + "reward_before_mean": -0.14269733056426048, + "reward_before_std": 0.3910446595400572, + "reward_change_max": 0.0, + "reward_change_mean": -0.24239975214004517, + "reward_change_min": -0.4103321433067322, + "reward_change_std": 0.1474100910127163, + "reward_std": 0.4501145798712969, + "rewards/accuracy_reward": 0.06250000186264515, + "rewards/cosine_scaled_reward": -0.20519732707180083, + "step": 29 + }, + { + "clip_fraction": 0.0, + "completion_length": 2777.1042404174805, + "epoch": 0.03428571428571429, + "grad_norm": 0.0785035490989685, + "kl": 4.0784478187561035e-05, + "lambda_div_used": 0.616574227809906, + "learning_rate": 9.944597532678119e-07, + "loss": 0.0306, + "reward": 0.014547215774655342, + "reward_after_mean": 0.014547215774655342, + "reward_after_std": 0.6317574586719275, + "reward_before_mean": 0.40835407795384526, + "reward_before_std": 0.5376694360747933, + "reward_change_max": 0.0, + "reward_change_mean": -0.3938068598508835, + "reward_change_min": -0.5825920477509499, + "reward_change_std": 0.22909015510231256, + "reward_std": 0.631757466122508, + "rewards/accuracy_reward": 0.3125000037252903, + "rewards/cosine_scaled_reward": 0.09585406119003892, + "step": 30 + }, + { + "clip_fraction": 0.0, + "completion_length": 3100.9166870117188, + "epoch": 0.03542857142857143, + "grad_norm": 0.05876823514699936, + "kl": 5.622208118438721e-05, + "lambda_div_used": 0.5746422484517097, + "learning_rate": 9.931634888554935e-07, + "loss": -0.0301, + "reward": -0.373639321886003, + "reward_after_mean": -0.373639321886003, + "reward_after_std": 0.40832165256142616, + "reward_before_mean": -0.11681870371103287, + "reward_before_std": 0.3402246618643403, + "reward_change_max": 0.0, + "reward_change_mean": -0.2568206209689379, + "reward_change_min": -0.4000260457396507, + "reward_change_std": 0.149553001858294, + "reward_std": 0.4083216693252325, + "rewards/accuracy_reward": 0.08333333395421505, + "rewards/cosine_scaled_reward": -0.20015203207731247, + "step": 31 + }, + { + "clip_fraction": 0.0, + "completion_length": 2879.625030517578, + "epoch": 0.036571428571428574, + "grad_norm": 0.060988157987594604, + "kl": 4.863739013671875e-05, + "lambda_div_used": 0.583471342921257, + "learning_rate": 9.917322325514487e-07, + "loss": 0.0143, + "reward": -0.09911171346902847, + "reward_after_mean": -0.09911171346902847, + "reward_after_std": 0.46612042374908924, + "reward_before_mean": 0.2912287414073944, + "reward_before_std": 0.3850706424564123, + "reward_change_max": 0.0, + "reward_change_mean": -0.39034045673906803, + "reward_change_min": -0.5949138030409813, + "reward_change_std": 0.23070038296282291, + "reward_std": 0.46612043119966984, + "rewards/accuracy_reward": 0.2916666716337204, + "rewards/cosine_scaled_reward": -0.0004379265010356903, + "step": 32 + }, + { + "clip_fraction": 0.0, + "completion_length": 3136.750030517578, + "epoch": 0.037714285714285714, + "grad_norm": 0.07551854848861694, + "kl": 4.787743091583252e-05, + "lambda_div_used": 0.6242911070585251, + "learning_rate": 9.901664203302124e-07, + "loss": -0.0446, + "reward": -0.06627794913947582, + "reward_after_mean": -0.06627794913947582, + "reward_after_std": 0.5915219262242317, + "reward_before_mean": 0.24627447500824928, + "reward_before_std": 0.5734489392489195, + "reward_change_max": 0.0, + "reward_change_mean": -0.3125524502247572, + "reward_change_min": -0.5326853170990944, + "reward_change_std": 0.2051102453842759, + "reward_std": 0.5915219560265541, + "rewards/accuracy_reward": 0.25000000931322575, + "rewards/cosine_scaled_reward": -0.0037255147472023964, + "step": 33 + }, + { + "clip_fraction": 0.0, + "completion_length": 2379.541702270508, + "epoch": 0.038857142857142854, + "grad_norm": 0.07498405873775482, + "kl": 4.60892915725708e-05, + "lambda_div_used": 0.6475758850574493, + "learning_rate": 9.88466529153356e-07, + "loss": 0.0814, + "reward": 0.054736172780394554, + "reward_after_mean": 0.054736172780394554, + "reward_after_std": 0.7529329154640436, + "reward_before_mean": 0.4133173357695341, + "reward_before_std": 0.6894109938293695, + "reward_change_max": 0.0, + "reward_change_mean": -0.3585811499506235, + "reward_change_min": -0.6234904117882252, + "reward_change_std": 0.23517589084804058, + "reward_std": 0.7529329396784306, + "rewards/accuracy_reward": 0.3333333358168602, + "rewards/cosine_scaled_reward": 0.07998399529606104, + "step": 34 + }, + { + "clip_fraction": 0.0, + "completion_length": 3113.1875610351562, + "epoch": 0.04, + "grad_norm": 0.1044503003358841, + "kl": 5.8710575103759766e-05, + "lambda_div_used": 0.5754920393228531, + "learning_rate": 9.866330768241983e-07, + "loss": -0.0058, + "reward": -0.36803475581109524, + "reward_after_mean": -0.36803475581109524, + "reward_after_std": 0.4120363052934408, + "reward_before_mean": -0.1010703444480896, + "reward_before_std": 0.3434353759512305, + "reward_change_max": 0.0, + "reward_change_mean": -0.2669644057750702, + "reward_change_min": -0.42853060737252235, + "reward_change_std": 0.1540099997073412, + "reward_std": 0.41203631460666656, + "rewards/accuracy_reward": 0.06250000186264515, + "rewards/cosine_scaled_reward": -0.16357035003602505, + "step": 35 + }, + { + "clip_fraction": 0.0, + "completion_length": 3350.1666870117188, + "epoch": 0.04114285714285714, + "grad_norm": 0.07969332486391068, + "kl": 6.110966205596924e-05, + "lambda_div_used": 0.5744208693504333, + "learning_rate": 9.846666218300807e-07, + "loss": -0.0514, + "reward": -0.4380789175629616, + "reward_after_mean": -0.4380789175629616, + "reward_after_std": 0.41310197673738003, + "reward_before_mean": -0.20339234871789813, + "reward_before_std": 0.337322598323226, + "reward_change_max": 0.0, + "reward_change_mean": -0.23468656465411186, + "reward_change_min": -0.3448420464992523, + "reward_change_std": 0.12617942783981562, + "reward_std": 0.4131019860506058, + "rewards/accuracy_reward": 0.0416666679084301, + "rewards/cosine_scaled_reward": -0.24505900964140892, + "step": 36 + }, + { + "clip_fraction": 0.0, + "completion_length": 3394.3958435058594, + "epoch": 0.04228571428571429, + "grad_norm": 0.05401439592242241, + "kl": 4.9054622650146484e-05, + "lambda_div_used": 0.5599935948848724, + "learning_rate": 9.825677631722435e-07, + "loss": 0.0086, + "reward": -0.3875775597989559, + "reward_after_mean": -0.3875775597989559, + "reward_after_std": 0.32731432281434536, + "reward_before_mean": -0.10551332868635654, + "reward_before_std": 0.2694389373064041, + "reward_change_max": 0.0, + "reward_change_mean": -0.28206423483788967, + "reward_change_min": -0.4370834305882454, + "reward_change_std": 0.16284553799778223, + "reward_std": 0.3273143321275711, + "rewards/accuracy_reward": 0.0833333358168602, + "rewards/cosine_scaled_reward": -0.1888466626405716, + "step": 37 + }, + { + "clip_fraction": 0.0, + "completion_length": 3311.0833435058594, + "epoch": 0.04342857142857143, + "grad_norm": 0.05246276035904884, + "kl": 4.945695400238037e-05, + "lambda_div_used": 0.5770035535097122, + "learning_rate": 9.80337140183366e-07, + "loss": -0.0117, + "reward": -0.3118477761745453, + "reward_after_mean": -0.3118477761745453, + "reward_after_std": 0.38849758356809616, + "reward_before_mean": -0.021045896457508206, + "reward_before_std": 0.3480216721072793, + "reward_change_max": 0.0, + "reward_change_mean": -0.2908018734306097, + "reward_change_min": -0.44894198328256607, + "reward_change_std": 0.17141664400696754, + "reward_std": 0.3884976040571928, + "rewards/accuracy_reward": 0.12500000558793545, + "rewards/cosine_scaled_reward": -0.14604590460658073, + "step": 38 + }, + { + "clip_fraction": 0.0, + "completion_length": 2919.604179382324, + "epoch": 0.044571428571428574, + "grad_norm": 0.08766351640224457, + "kl": 3.2648444175720215e-05, + "lambda_div_used": 0.5717808604240417, + "learning_rate": 9.779754323328192e-07, + "loss": 0.032, + "reward": -0.14589639008045197, + "reward_after_mean": -0.14589639008045197, + "reward_after_std": 0.4250806160271168, + "reward_before_mean": 0.26390516571700573, + "reward_before_std": 0.3228347860276699, + "reward_change_max": 0.0, + "reward_change_mean": -0.40980157628655434, + "reward_change_min": -0.5692479386925697, + "reward_change_std": 0.22337355464696884, + "reward_std": 0.42508063092827797, + "rewards/accuracy_reward": 0.25000000558793545, + "rewards/cosine_scaled_reward": 0.013905153144150972, + "step": 39 + }, + { + "clip_fraction": 0.0, + "completion_length": 2468.7916870117188, + "epoch": 0.045714285714285714, + "grad_norm": 0.07357986271381378, + "kl": 4.18052077293396e-05, + "lambda_div_used": 0.5706272348761559, + "learning_rate": 9.754833590196926e-07, + "loss": 0.0148, + "reward": -0.2578982161357999, + "reward_after_mean": -0.2578982161357999, + "reward_after_std": 0.4464099854230881, + "reward_before_mean": 0.1052470114082098, + "reward_before_std": 0.31700289947912097, + "reward_change_max": 0.0, + "reward_change_mean": -0.3631452303379774, + "reward_change_min": -0.5201860442757607, + "reward_change_std": 0.1936064399778843, + "reward_std": 0.4464099947363138, + "rewards/accuracy_reward": 0.1666666679084301, + "rewards/cosine_scaled_reward": -0.06141966814175248, + "step": 40 + }, + { + "clip_fraction": 0.0, + "completion_length": 3203.312515258789, + "epoch": 0.046857142857142854, + "grad_norm": 0.05290238931775093, + "kl": 5.0440430641174316e-05, + "lambda_div_used": 0.6480180770158768, + "learning_rate": 9.728616793536587e-07, + "loss": -0.0199, + "reward": -0.04493038635700941, + "reward_after_mean": -0.04493038635700941, + "reward_after_std": 0.7092454191297293, + "reward_before_mean": 0.23497815802693367, + "reward_before_std": 0.693053056485951, + "reward_change_max": 0.0, + "reward_change_mean": -0.2799085471779108, + "reward_change_min": -0.5064845345914364, + "reward_change_std": 0.19558908697217703, + "reward_std": 0.7092454340308905, + "rewards/accuracy_reward": 0.2291666716337204, + "rewards/cosine_scaled_reward": 0.005811510724015534, + "step": 41 + }, + { + "clip_fraction": 0.0, + "completion_length": 2890.9791927337646, + "epoch": 0.048, + "grad_norm": 0.09494666010141373, + "kl": 5.85019588470459e-05, + "lambda_div_used": 0.5329261645674706, + "learning_rate": 9.701111919237408e-07, + "loss": 0.0335, + "reward": -0.5650591850280762, + "reward_after_mean": -0.5650591850280762, + "reward_after_std": 0.225538931787014, + "reward_before_mean": -0.32472414150834084, + "reward_before_std": 0.14451794046908617, + "reward_change_max": 0.0, + "reward_change_mean": -0.24033503979444504, + "reward_change_min": -0.358134388923645, + "reward_change_std": 0.12634030357003212, + "reward_std": 0.2255389392375946, + "rewards/accuracy_reward": 0.0, + "rewards/cosine_scaled_reward": -0.32472414895892143, + "step": 42 + }, + { + "clip_fraction": 0.0, + "completion_length": 3027.9583587646484, + "epoch": 0.04914285714285714, + "grad_norm": 0.05899634212255478, + "kl": 4.252791404724121e-05, + "lambda_div_used": 0.595756284892559, + "learning_rate": 9.672327345550543e-07, + "loss": -0.017, + "reward": -0.31989429891109467, + "reward_after_mean": -0.31989429891109467, + "reward_after_std": 0.5033334400504827, + "reward_before_mean": -0.07388713955879211, + "reward_before_std": 0.43908379040658474, + "reward_change_max": 0.0, + "reward_change_mean": -0.24600715190172195, + "reward_change_min": -0.4015818126499653, + "reward_change_std": 0.143897395581007, + "reward_std": 0.5033334512263536, + "rewards/accuracy_reward": 0.1041666679084301, + "rewards/cosine_scaled_reward": -0.1780538223683834, + "step": 43 + }, + { + "clip_fraction": 0.0, + "completion_length": 2372.5833587646484, + "epoch": 0.05028571428571429, + "grad_norm": 0.08952542394399643, + "kl": 4.1332095861434937e-05, + "lambda_div_used": 0.5768283307552338, + "learning_rate": 9.64227184053598e-07, + "loss": 0.0238, + "reward": -0.09475147165358067, + "reward_after_mean": -0.09475147165358067, + "reward_after_std": 0.4600023180246353, + "reward_before_mean": 0.33489263616502285, + "reward_before_std": 0.35124383692163974, + "reward_change_max": 0.0, + "reward_change_mean": -0.4296441040933132, + "reward_change_min": -0.6358921378850937, + "reward_change_std": 0.24556122440844774, + "reward_std": 0.4600023254752159, + "rewards/accuracy_reward": 0.2708333395421505, + "rewards/cosine_scaled_reward": 0.06405930174514651, + "step": 44 + }, + { + "clip_fraction": 0.0, + "completion_length": 3178.125, + "epoch": 0.05142857142857143, + "grad_norm": 0.05793697386980057, + "kl": 4.953145980834961e-05, + "lambda_div_used": 0.6092750132083893, + "learning_rate": 9.610954559391704e-07, + "loss": 0.0375, + "reward": -0.24964951165020466, + "reward_after_mean": -0.24964951165020466, + "reward_after_std": 0.575963044539094, + "reward_before_mean": 0.010031561367213726, + "reward_before_std": 0.5040898034349084, + "reward_change_max": 0.0, + "reward_change_mean": -0.25968106649816036, + "reward_change_min": -0.41585399955511093, + "reward_change_std": 0.15605208091437817, + "reward_std": 0.5759630724787712, + "rewards/accuracy_reward": 0.12500000186264515, + "rewards/cosine_scaled_reward": -0.114968441426754, + "step": 45 + }, + { + "clip_fraction": 0.0, + "completion_length": 3160.895866394043, + "epoch": 0.052571428571428575, + "grad_norm": 0.06750793755054474, + "kl": 4.884600639343262e-05, + "lambda_div_used": 0.5543409436941147, + "learning_rate": 9.578385041664925e-07, + "loss": 0.0442, + "reward": -0.47163831628859043, + "reward_after_mean": -0.47163831628859043, + "reward_after_std": 0.32501144520938396, + "reward_before_mean": -0.21356616588309407, + "reward_before_std": 0.24316862598061562, + "reward_change_max": 0.0, + "reward_change_mean": -0.25807216577231884, + "reward_change_min": -0.3883417770266533, + "reward_change_std": 0.13798328768461943, + "reward_std": 0.3250114619731903, + "rewards/accuracy_reward": 0.02083333395421505, + "rewards/cosine_scaled_reward": -0.23439949192106724, + "step": 46 + }, + { + "clip_fraction": 0.0, + "completion_length": 2624.270881652832, + "epoch": 0.053714285714285714, + "grad_norm": 0.10712098330259323, + "kl": 5.3569674491882324e-05, + "lambda_div_used": 0.6219898834824562, + "learning_rate": 9.54457320834625e-07, + "loss": 0.0102, + "reward": -0.10781855694949627, + "reward_after_mean": -0.10781855694949627, + "reward_after_std": 0.5886904541403055, + "reward_before_mean": 0.19626505579799414, + "reward_before_std": 0.5641138087958097, + "reward_change_max": 0.0, + "reward_change_mean": -0.3040836229920387, + "reward_change_min": -0.5314439944922924, + "reward_change_std": 0.2001748401671648, + "reward_std": 0.5886904690414667, + "rewards/accuracy_reward": 0.22916667349636555, + "rewards/cosine_scaled_reward": -0.03290162514895201, + "step": 47 + }, + { + "clip_fraction": 0.0, + "completion_length": 2589.458396911621, + "epoch": 0.054857142857142854, + "grad_norm": 0.07933427393436432, + "kl": 4.4988468289375305e-05, + "lambda_div_used": 0.6262567341327667, + "learning_rate": 9.509529358847654e-07, + "loss": -0.0358, + "reward": -0.13860276085324585, + "reward_after_mean": -0.13860276085324585, + "reward_after_std": 0.5960155855864286, + "reward_before_mean": 0.13786590658128262, + "reward_before_std": 0.5872014760971069, + "reward_change_max": 0.0, + "reward_change_mean": -0.2764686793088913, + "reward_change_min": -0.5099791921675205, + "reward_change_std": 0.19576375279575586, + "reward_std": 0.5960155855864286, + "rewards/accuracy_reward": 0.20833333767950535, + "rewards/cosine_scaled_reward": -0.07046743389219046, + "step": 48 + }, + { + "clip_fraction": 0.0, + "completion_length": 1985.2291946411133, + "epoch": 0.056, + "grad_norm": 0.12161526083946228, + "kl": 4.811584949493408e-05, + "lambda_div_used": 0.5859075263142586, + "learning_rate": 9.473264167865171e-07, + "loss": 0.0065, + "reward": -0.21219712868332863, + "reward_after_mean": -0.21219712868332863, + "reward_after_std": 0.439599821344018, + "reward_before_mean": 0.10536342021077871, + "reward_before_std": 0.39430073963012546, + "reward_change_max": 0.0, + "reward_change_mean": -0.3175605833530426, + "reward_change_min": -0.4657192714512348, + "reward_change_std": 0.19229275174438953, + "reward_std": 0.4395998399704695, + "rewards/accuracy_reward": 0.20833334140479565, + "rewards/cosine_scaled_reward": -0.10296990955248475, + "step": 49 + }, + { + "clip_fraction": 0.0, + "completion_length": 2880.250011444092, + "epoch": 0.05714285714285714, + "grad_norm": 0.07723147422075272, + "kl": 3.2588839530944824e-05, + "lambda_div_used": 0.5741998106241226, + "learning_rate": 9.43578868212728e-07, + "loss": 0.0182, + "reward": -0.0913238637149334, + "reward_after_mean": -0.0913238637149334, + "reward_after_std": 0.42474013939499855, + "reward_before_mean": 0.3248649761080742, + "reward_before_std": 0.3397149038501084, + "reward_change_max": 0.0, + "reward_change_mean": -0.41618884168565273, + "reward_change_min": -0.6034708395600319, + "reward_change_std": 0.23950859624892473, + "reward_std": 0.4247401561588049, + "rewards/accuracy_reward": 0.3125000074505806, + "rewards/cosine_scaled_reward": 0.012364983558654785, + "step": 50 + }, + { + "clip_fraction": 0.0, + "completion_length": 2433.020854949951, + "epoch": 0.05828571428571429, + "grad_norm": 0.09581972658634186, + "kl": 5.4717063903808594e-05, + "lambda_div_used": 0.5894461125135422, + "learning_rate": 9.397114317029974e-07, + "loss": -0.0308, + "reward": -0.28266510320827365, + "reward_after_mean": -0.28266510320827365, + "reward_after_std": 0.44258156418800354, + "reward_before_mean": 0.012647990137338638, + "reward_before_std": 0.40818152111023664, + "reward_change_max": 0.0, + "reward_change_mean": -0.29531308077275753, + "reward_change_min": -0.4761137217283249, + "reward_change_std": 0.1856073010712862, + "reward_std": 0.4425815735012293, + "rewards/accuracy_reward": 0.16666666977107525, + "rewards/cosine_scaled_reward": -0.15401867777109146, + "step": 51 + }, + { + "clip_fraction": 0.0, + "completion_length": 2707.250030517578, + "epoch": 0.05942857142857143, + "grad_norm": 0.14702580869197845, + "kl": 4.030205309391022e-05, + "lambda_div_used": 0.6197419166564941, + "learning_rate": 9.357252853159505e-07, + "loss": 0.0298, + "reward": 0.17821598052978516, + "reward_after_mean": 0.17821598052978516, + "reward_after_std": 0.6063419748097658, + "reward_before_mean": 0.629100788384676, + "reward_before_std": 0.5578461596742272, + "reward_change_max": 0.0, + "reward_change_mean": -0.4508847985416651, + "reward_change_min": -0.6935725994408131, + "reward_change_std": 0.2830996550619602, + "reward_std": 0.6063419822603464, + "rewards/accuracy_reward": 0.4583333432674408, + "rewards/cosine_scaled_reward": 0.17076744884252548, + "step": 52 + }, + { + "clip_fraction": 0.0, + "completion_length": 2799.8125610351562, + "epoch": 0.060571428571428575, + "grad_norm": 0.07566332817077637, + "kl": 5.0321221351623535e-05, + "lambda_div_used": 0.6389462202787399, + "learning_rate": 9.316216432703916e-07, + "loss": 0.0088, + "reward": -0.0043678306974470615, + "reward_after_mean": -0.0043678306974470615, + "reward_after_std": 0.6600480955094099, + "reward_before_mean": 0.31353550031781197, + "reward_before_std": 0.6461421558633447, + "reward_change_max": 0.0, + "reward_change_mean": -0.3179033286869526, + "reward_change_min": -0.5801984183490276, + "reward_change_std": 0.22101250104606152, + "reward_std": 0.6600481104105711, + "rewards/accuracy_reward": 0.29166667722165585, + "rewards/cosine_scaled_reward": 0.0218688128516078, + "step": 53 + }, + { + "clip_fraction": 0.0, + "completion_length": 2297.9583778381348, + "epoch": 0.061714285714285715, + "grad_norm": 0.09625467658042908, + "kl": 4.2378902435302734e-05, + "lambda_div_used": 0.6194662973284721, + "learning_rate": 9.274017555754407e-07, + "loss": 0.0224, + "reward": 0.0802488662302494, + "reward_after_mean": 0.0802488662302494, + "reward_after_std": 0.6323203574866056, + "reward_before_mean": 0.4949483387172222, + "reward_before_std": 0.5504263024777174, + "reward_change_max": 0.0, + "reward_change_mean": -0.41469944082200527, + "reward_change_min": -0.6165112145245075, + "reward_change_std": 0.2416615542024374, + "reward_std": 0.632320374250412, + "rewards/accuracy_reward": 0.3541666753590107, + "rewards/cosine_scaled_reward": 0.1407816605642438, + "step": 54 + }, + { + "clip_fraction": 0.0, + "completion_length": 2896.416702270508, + "epoch": 0.06285714285714286, + "grad_norm": 0.05907125398516655, + "kl": 3.5256147384643555e-05, + "lambda_div_used": 0.602072462439537, + "learning_rate": 9.230669076497687e-07, + "loss": 0.0467, + "reward": -0.13429339788854122, + "reward_after_mean": -0.13429339788854122, + "reward_after_std": 0.57946902140975, + "reward_before_mean": 0.22700263070873916, + "reward_before_std": 0.4767341245897114, + "reward_change_max": 0.0, + "reward_change_mean": -0.36129603534936905, + "reward_change_min": -0.5782174952328205, + "reward_change_std": 0.21661545429378748, + "reward_std": 0.5794690307229757, + "rewards/accuracy_reward": 0.25000000186264515, + "rewards/cosine_scaled_reward": -0.02299738209694624, + "step": 55 + }, + { + "clip_fraction": 0.0, + "completion_length": 3075.187545776367, + "epoch": 0.064, + "grad_norm": 0.06310205906629562, + "kl": 4.1544437408447266e-05, + "lambda_div_used": 0.5549901723861694, + "learning_rate": 9.186184199300463e-07, + "loss": -0.0108, + "reward": -0.4740895018912852, + "reward_after_mean": -0.4740895018912852, + "reward_after_std": 0.32212498411536217, + "reward_before_mean": -0.2245362438261509, + "reward_before_std": 0.24502124171704054, + "reward_change_max": 0.0, + "reward_change_mean": -0.24955323711037636, + "reward_change_min": -0.3650141842663288, + "reward_change_std": 0.13405024446547031, + "reward_std": 0.3221249897032976, + "rewards/accuracy_reward": 0.02083333395421505, + "rewards/cosine_scaled_reward": -0.24536957405507565, + "step": 56 + }, + { + "clip_fraction": 0.0, + "completion_length": 3013.8541870117188, + "epoch": 0.06514285714285714, + "grad_norm": 0.052018001675605774, + "kl": 3.096461296081543e-05, + "lambda_div_used": 0.5674270242452621, + "learning_rate": 9.140576474687263e-07, + "loss": 0.0192, + "reward": -0.45193217881023884, + "reward_after_mean": -0.45193217881023884, + "reward_after_std": 0.37017360515892506, + "reward_before_mean": -0.21255622059106827, + "reward_before_std": 0.3090948835015297, + "reward_change_max": 0.0, + "reward_change_mean": -0.23937593773007393, + "reward_change_min": -0.42212119325995445, + "reward_change_std": 0.14615233521908522, + "reward_std": 0.3701736144721508, + "rewards/accuracy_reward": 0.0416666679084301, + "rewards/cosine_scaled_reward": -0.2542228833772242, + "step": 57 + }, + { + "clip_fraction": 0.0, + "completion_length": 2257.8750534057617, + "epoch": 0.06628571428571428, + "grad_norm": 0.09038142114877701, + "kl": 3.3486634492874146e-05, + "lambda_div_used": 0.6153116375207901, + "learning_rate": 9.093859795212817e-07, + "loss": 0.049, + "reward": -0.07369695231318474, + "reward_after_mean": -0.07369695231318474, + "reward_after_std": 0.5549236796796322, + "reward_before_mean": 0.2532845102250576, + "reward_before_std": 0.5286985114216805, + "reward_change_max": 0.0, + "reward_change_mean": -0.3269814867526293, + "reward_change_min": -0.537376407533884, + "reward_change_std": 0.20782853197306395, + "reward_std": 0.5549236983060837, + "rewards/accuracy_reward": 0.27083334513008595, + "rewards/cosine_scaled_reward": -0.017548808827996254, + "step": 58 + }, + { + "clip_fraction": 0.0, + "completion_length": 2807.6250381469727, + "epoch": 0.06742857142857143, + "grad_norm": 0.06863339245319366, + "kl": 3.577768802642822e-05, + "lambda_div_used": 0.6041718497872353, + "learning_rate": 9.046048391230247e-07, + "loss": -0.005, + "reward": -0.25026911310851574, + "reward_after_mean": -0.25026911310851574, + "reward_after_std": 0.5205795764923096, + "reward_before_mean": 0.026796480640769005, + "reward_before_std": 0.4837551396340132, + "reward_change_max": 0.0, + "reward_change_mean": -0.27706561237573624, + "reward_change_min": -0.5239802338182926, + "reward_change_std": 0.18853904772549868, + "reward_std": 0.5205795876681805, + "rewards/accuracy_reward": 0.1458333358168602, + "rewards/cosine_scaled_reward": -0.11903684702701867, + "step": 59 + }, + { + "clip_fraction": 0.0, + "completion_length": 2797.5833740234375, + "epoch": 0.06857142857142857, + "grad_norm": 0.07598927617073059, + "kl": 3.738701343536377e-05, + "lambda_div_used": 0.6269034072756767, + "learning_rate": 8.997156826556369e-07, + "loss": 0.0313, + "reward": -0.2369612492620945, + "reward_after_mean": -0.2369612492620945, + "reward_after_std": 0.6222166065126657, + "reward_before_mean": 0.005125788040459156, + "reward_before_std": 0.5912308068946004, + "reward_change_max": 0.0, + "reward_change_mean": -0.24208704754710197, + "reward_change_min": -0.48471353203058243, + "reward_change_std": 0.17278119549155235, + "reward_std": 0.6222166288644075, + "rewards/accuracy_reward": 0.14583333767950535, + "rewards/cosine_scaled_reward": -0.14070754405111074, + "step": 60 + }, + { + "clip_fraction": 0.0, + "completion_length": 2917.812515258789, + "epoch": 0.06971428571428571, + "grad_norm": 0.06223488971590996, + "kl": 3.0603259801864624e-05, + "lambda_div_used": 0.6177623867988586, + "learning_rate": 8.9471999940354e-07, + "loss": 0.0259, + "reward": -0.1357121616601944, + "reward_after_mean": -0.1357121616601944, + "reward_after_std": 0.5694319568574429, + "reward_before_mean": 0.16569078899919987, + "reward_before_std": 0.5441586868837476, + "reward_change_max": 0.0, + "reward_change_mean": -0.3014029450714588, + "reward_change_min": -0.5209132842719555, + "reward_change_std": 0.20053375512361526, + "reward_std": 0.5694319736212492, + "rewards/accuracy_reward": 0.20833333767950535, + "rewards/cosine_scaled_reward": -0.042642548214644194, + "step": 61 + }, + { + "clip_fraction": 0.0, + "completion_length": 2507.8958854675293, + "epoch": 0.07085714285714285, + "grad_norm": 0.0747612789273262, + "kl": 2.6777386665344238e-05, + "lambda_div_used": 0.61732517182827, + "learning_rate": 8.896193111002475e-07, + "loss": 0.0366, + "reward": 0.010478481650352478, + "reward_after_mean": 0.010478481650352478, + "reward_after_std": 0.6334415413439274, + "reward_before_mean": 0.4018659461289644, + "reward_before_std": 0.5490978918969631, + "reward_change_max": 0.0, + "reward_change_mean": -0.39138743840157986, + "reward_change_min": -0.6315983533859253, + "reward_change_std": 0.24404066987335682, + "reward_std": 0.6334415581077337, + "rewards/accuracy_reward": 0.33333333767950535, + "rewards/cosine_scaled_reward": 0.06853258889168501, + "step": 62 + }, + { + "clip_fraction": 0.0, + "completion_length": 1902.5000228881836, + "epoch": 0.072, + "grad_norm": 0.07997458428144455, + "kl": 2.5197863578796387e-05, + "lambda_div_used": 0.6046535074710846, + "learning_rate": 8.844151714648274e-07, + "loss": 0.059, + "reward": 0.0217137411236763, + "reward_after_mean": 0.0217137411236763, + "reward_after_std": 0.5547807831317186, + "reward_before_mean": 0.42593196779489517, + "reward_before_std": 0.49076249497011304, + "reward_change_max": 0.0, + "reward_change_mean": -0.4042182229459286, + "reward_change_min": -0.6314646527171135, + "reward_change_std": 0.25159123074263334, + "reward_std": 0.5547807849943638, + "rewards/accuracy_reward": 0.3541666716337204, + "rewards/cosine_scaled_reward": 0.07176528126001358, + "step": 63 + }, + { + "clip_fraction": 0.0, + "completion_length": 2790.979217529297, + "epoch": 0.07314285714285715, + "grad_norm": 0.07697053253650665, + "kl": 4.419684410095215e-05, + "lambda_div_used": 0.6203151121735573, + "learning_rate": 8.791091657286267e-07, + "loss": 0.061, + "reward": -0.1100537832826376, + "reward_after_mean": -0.1100537832826376, + "reward_after_std": 0.5783387050032616, + "reward_before_mean": 0.19534806534647942, + "reward_before_std": 0.5548522733151913, + "reward_change_max": 0.0, + "reward_change_mean": -0.3054018337279558, + "reward_change_min": -0.5362299680709839, + "reward_change_std": 0.2037976048886776, + "reward_std": 0.5783387236297131, + "rewards/accuracy_reward": 0.20833333767950535, + "rewards/cosine_scaled_reward": -0.012985273322556168, + "step": 64 + }, + { + "clip_fraction": 0.0, + "completion_length": 2605.2708587646484, + "epoch": 0.07428571428571429, + "grad_norm": 0.0772913321852684, + "kl": 3.089010715484619e-05, + "lambda_div_used": 0.5563548430800438, + "learning_rate": 8.737029101523929e-07, + "loss": -0.0861, + "reward": -0.33125742711126804, + "reward_after_mean": -0.33125742711126804, + "reward_after_std": 0.3626741226762533, + "reward_before_mean": 0.005530592054128647, + "reward_before_std": 0.2566492212936282, + "reward_change_max": 0.0, + "reward_change_mean": -0.33678802102804184, + "reward_change_min": -0.5241431556642056, + "reward_change_std": 0.18918895348906517, + "reward_std": 0.36267412453889847, + "rewards/accuracy_reward": 0.1666666679084301, + "rewards/cosine_scaled_reward": -0.161136067006737, + "step": 65 + }, + { + "clip_fraction": 0.0, + "completion_length": 2088.562511444092, + "epoch": 0.07542857142857143, + "grad_norm": 0.10951534658670425, + "kl": 2.690870314836502e-05, + "lambda_div_used": 0.5969364494085312, + "learning_rate": 8.681980515339463e-07, + "loss": 0.0287, + "reward": 0.012385480105876923, + "reward_after_mean": 0.012385480105876923, + "reward_after_std": 0.5128760654479265, + "reward_before_mean": 0.44090043008327484, + "reward_before_std": 0.4437296399846673, + "reward_change_max": 0.0, + "reward_change_mean": -0.4285149369388819, + "reward_change_min": -0.660201895982027, + "reward_change_std": 0.25701938942074776, + "reward_std": 0.512876084074378, + "rewards/accuracy_reward": 0.3750000111758709, + "rewards/cosine_scaled_reward": 0.0659004095941782, + "step": 66 + }, + { + "clip_fraction": 0.0, + "completion_length": 3402.8958740234375, + "epoch": 0.07657142857142857, + "grad_norm": 0.047973256558179855, + "kl": 3.771483898162842e-05, + "lambda_div_used": 0.5712955147027969, + "learning_rate": 8.625962667065487e-07, + "loss": -0.0027, + "reward": -0.4502652711234987, + "reward_after_mean": -0.4502652711234987, + "reward_after_std": 0.39950996078550816, + "reward_before_mean": -0.22067994717508554, + "reward_before_std": 0.3211147477850318, + "reward_change_max": 0.0, + "reward_change_mean": -0.229585325345397, + "reward_change_min": -0.3470626436173916, + "reward_change_std": 0.12443333957344294, + "reward_std": 0.39950997941195965, + "rewards/accuracy_reward": 0.0416666679084301, + "rewards/cosine_scaled_reward": -0.26234661415219307, + "step": 67 + }, + { + "clip_fraction": 0.0, + "completion_length": 1642.9166984558105, + "epoch": 0.07771428571428571, + "grad_norm": 0.11412619799375534, + "kl": 2.551823854446411e-05, + "lambda_div_used": 0.6189040914177895, + "learning_rate": 8.568992620281243e-07, + "loss": -0.0912, + "reward": -0.22925877509987913, + "reward_after_mean": -0.22925877509987913, + "reward_after_std": 0.6055262424051762, + "reward_before_mean": 0.03262739907950163, + "reward_before_std": 0.5506744375452399, + "reward_change_max": 0.0, + "reward_change_mean": -0.2618861813098192, + "reward_change_min": -0.4676270857453346, + "reward_change_std": 0.16872322466224432, + "reward_std": 0.6055262610316277, + "rewards/accuracy_reward": 0.1250000037252903, + "rewards/cosine_scaled_reward": -0.09237260185182095, + "step": 68 + }, + { + "clip_fraction": 0.0, + "completion_length": 2018.6666946411133, + "epoch": 0.07885714285714286, + "grad_norm": 0.11094633489847183, + "kl": 4.1466206312179565e-05, + "lambda_div_used": 0.5734386518597603, + "learning_rate": 8.511087728614862e-07, + "loss": -0.0618, + "reward": -0.4316765144467354, + "reward_after_mean": -0.4316765144467354, + "reward_after_std": 0.4006939698010683, + "reward_before_mean": -0.1882182292174548, + "reward_before_std": 0.331131674349308, + "reward_change_max": 0.0, + "reward_change_mean": -0.24345828033983707, + "reward_change_min": -0.3763142116367817, + "reward_change_std": 0.13501812983304262, + "reward_std": 0.40069398283958435, + "rewards/accuracy_reward": 0.0416666679084301, + "rewards/cosine_scaled_reward": -0.22988490015268326, + "step": 69 + }, + { + "clip_fraction": 0.0, + "completion_length": 2860.8958587646484, + "epoch": 0.08, + "grad_norm": 0.06797178089618683, + "kl": 4.0102750062942505e-05, + "lambda_div_used": 0.6089852601289749, + "learning_rate": 8.452265630457282e-07, + "loss": 0.0416, + "reward": -0.16451344639062881, + "reward_after_mean": -0.16451344639062881, + "reward_after_std": 0.5175492316484451, + "reward_before_mean": 0.1350960824638605, + "reward_before_std": 0.5058744940906763, + "reward_change_max": 0.0, + "reward_change_mean": -0.2996095381677151, + "reward_change_min": -0.5449608005583286, + "reward_change_std": 0.2059446070343256, + "reward_std": 0.5175492409616709, + "rewards/accuracy_reward": 0.1875000037252903, + "rewards/cosine_scaled_reward": -0.05240389332175255, + "step": 70 + }, + { + "clip_fraction": 0.0, + "completion_length": 2700.2708587646484, + "epoch": 0.08114285714285714, + "grad_norm": 0.07971946895122528, + "kl": 4.7519803047180176e-05, + "lambda_div_used": 0.6114047914743423, + "learning_rate": 8.392544243589427e-07, + "loss": 0.0248, + "reward": -0.17357509583234787, + "reward_after_mean": -0.17357509583234787, + "reward_after_std": 0.5260436423122883, + "reward_before_mean": 0.1154884397983551, + "reward_before_std": 0.5224206217098981, + "reward_change_max": 0.0, + "reward_change_mean": -0.2890635374933481, + "reward_change_min": -0.5293963067233562, + "reward_change_std": 0.20269155222922564, + "reward_std": 0.5260436479002237, + "rewards/accuracy_reward": 0.1666666679084301, + "rewards/cosine_scaled_reward": -0.051178233698010445, + "step": 71 + }, + { + "clip_fraction": 0.0, + "completion_length": 2651.1250610351562, + "epoch": 0.08228571428571428, + "grad_norm": 0.1173771321773529, + "kl": 5.626678466796875e-05, + "lambda_div_used": 0.5625430718064308, + "learning_rate": 8.331941759724268e-07, + "loss": -0.0294, + "reward": -0.4364648088812828, + "reward_after_mean": -0.4364648088812828, + "reward_after_std": 0.3423657324165106, + "reward_before_mean": -0.17360325902700424, + "reward_before_std": 0.28492841869592667, + "reward_change_max": 0.0, + "reward_change_mean": -0.2628615368157625, + "reward_change_min": -0.44537778943777084, + "reward_change_std": 0.15708798076957464, + "reward_std": 0.3423657491803169, + "rewards/accuracy_reward": 0.0416666679084301, + "rewards/cosine_scaled_reward": -0.2152699390426278, + "step": 72 + }, + { + "clip_fraction": 0.0, + "completion_length": 3339.9583740234375, + "epoch": 0.08342857142857144, + "grad_norm": 0.04790791496634483, + "kl": 4.526972770690918e-05, + "lambda_div_used": 0.6265770718455315, + "learning_rate": 8.270476638965461e-07, + "loss": 0.0626, + "reward": -0.12956268154084682, + "reward_after_mean": -0.12956268154084682, + "reward_after_std": 0.6143265012651682, + "reward_before_mean": 0.15906396182253957, + "reward_before_std": 0.5886987801641226, + "reward_change_max": 0.0, + "reward_change_mean": -0.288626654073596, + "reward_change_min": -0.5101466961205006, + "reward_change_std": 0.1920691430568695, + "reward_std": 0.6143265273422003, + "rewards/accuracy_reward": 0.20833333767950535, + "rewards/cosine_scaled_reward": -0.049269367940723896, + "step": 73 + }, + { + "clip_fraction": 0.0, + "completion_length": 2385.7500381469727, + "epoch": 0.08457142857142858, + "grad_norm": 0.06600002944469452, + "kl": 3.542378544807434e-05, + "lambda_div_used": 0.6220900639891624, + "learning_rate": 8.208167604184217e-07, + "loss": -0.0312, + "reward": -0.0998319232603535, + "reward_after_mean": -0.0998319232603535, + "reward_after_std": 0.5906735248863697, + "reward_before_mean": 0.20307225361466408, + "reward_before_std": 0.566557977348566, + "reward_change_max": 0.0, + "reward_change_mean": -0.30290416814386845, + "reward_change_min": -0.5291841961443424, + "reward_change_std": 0.20113296527415514, + "reward_std": 0.5906735453754663, + "rewards/accuracy_reward": 0.20833333767950535, + "rewards/cosine_scaled_reward": -0.005261080645141192, + "step": 74 + }, + { + "clip_fraction": 0.0, + "completion_length": 2732.458335876465, + "epoch": 0.08571428571428572, + "grad_norm": 0.059586018323898315, + "kl": 3.4804921597242355e-05, + "lambda_div_used": 0.5497717335820198, + "learning_rate": 8.145033635316128e-07, + "loss": -0.026, + "reward": -0.16143636964261532, + "reward_after_mean": -0.16143636964261532, + "reward_after_std": 0.41724423691630363, + "reward_before_mean": 0.31992355175316334, + "reward_before_std": 0.22394374571740627, + "reward_change_max": 0.0, + "reward_change_mean": -0.4813598971813917, + "reward_change_min": -0.6314556114375591, + "reward_change_std": 0.24685372970998287, + "reward_std": 0.4172442499548197, + "rewards/accuracy_reward": 0.27083333395421505, + "rewards/cosine_scaled_reward": 0.0490901879966259, + "step": 75 + }, + { + "clip_fraction": 0.0, + "completion_length": 2649.9791870117188, + "epoch": 0.08685714285714285, + "grad_norm": 0.07793173938989639, + "kl": 3.8176774978637695e-05, + "lambda_div_used": 0.5533522665500641, + "learning_rate": 8.081093963579707e-07, + "loss": 0.0329, + "reward": -0.4524771338328719, + "reward_after_mean": -0.4524771338328719, + "reward_after_std": 0.32186589390039444, + "reward_before_mean": -0.18054483737796545, + "reward_before_std": 0.23863591719418764, + "reward_change_max": 0.0, + "reward_change_mean": -0.2719323057681322, + "reward_change_min": -0.402019876986742, + "reward_change_std": 0.14489794615656137, + "reward_std": 0.32186589762568474, + "rewards/accuracy_reward": 0.02083333395421505, + "rewards/cosine_scaled_reward": -0.20137817226350307, + "step": 76 + }, + { + "clip_fraction": 0.0, + "completion_length": 2847.8333587646484, + "epoch": 0.088, + "grad_norm": 0.06544006615877151, + "kl": 4.89354133605957e-05, + "lambda_div_used": 0.5558431893587112, + "learning_rate": 8.01636806561836e-07, + "loss": -0.0184, + "reward": -0.46523131616413593, + "reward_after_mean": -0.46523131616413593, + "reward_after_std": 0.3304946720600128, + "reward_before_mean": -0.20099198445677757, + "reward_before_std": 0.2519041560590267, + "reward_change_max": 0.0, + "reward_change_mean": -0.26423933170735836, + "reward_change_min": -0.37857379391789436, + "reward_change_std": 0.140173084102571, + "reward_std": 0.3304946757853031, + "rewards/accuracy_reward": 0.02083333395421505, + "rewards/cosine_scaled_reward": -0.22182531282305717, + "step": 77 + }, + { + "clip_fraction": 0.0, + "completion_length": 3269.8333740234375, + "epoch": 0.08914285714285715, + "grad_norm": 0.05014213174581528, + "kl": 3.695487976074219e-05, + "lambda_div_used": 0.6150171384215355, + "learning_rate": 7.950875657567621e-07, + "loss": -0.0153, + "reward": -0.23963370453566313, + "reward_after_mean": -0.23963370453566313, + "reward_after_std": 0.5818531475961208, + "reward_before_mean": 0.01525909942574799, + "reward_before_std": 0.5286716222763062, + "reward_change_max": 0.0, + "reward_change_mean": -0.2548927925527096, + "reward_change_min": -0.411530327051878, + "reward_change_std": 0.1550313262268901, + "reward_std": 0.5818531513214111, + "rewards/accuracy_reward": 0.12500000186264515, + "rewards/cosine_scaled_reward": -0.10974090336821973, + "step": 78 + }, + { + "clip_fraction": 0.0, + "completion_length": 2262.062511444092, + "epoch": 0.09028571428571429, + "grad_norm": 0.06913956999778748, + "kl": 3.7103891372680664e-05, + "lambda_div_used": 0.5935603529214859, + "learning_rate": 7.884636689049422e-07, + "loss": 0.0312, + "reward": -0.23656679573468864, + "reward_after_mean": -0.23656679573468864, + "reward_after_std": 0.5612265523523092, + "reward_before_mean": 0.08522074297070503, + "reward_before_std": 0.4243400124832988, + "reward_change_max": 0.0, + "reward_change_mean": -0.3217875510454178, + "reward_change_min": -0.4353032112121582, + "reward_change_std": 0.16354462038725615, + "reward_std": 0.5612265765666962, + "rewards/accuracy_reward": 0.18750000186264515, + "rewards/cosine_scaled_reward": -0.10227925609797239, + "step": 79 + }, + { + "clip_fraction": 0.0, + "completion_length": 3292.3333587646484, + "epoch": 0.09142857142857143, + "grad_norm": 0.0538838729262352, + "kl": 4.382804036140442e-05, + "lambda_div_used": 0.5688970535993576, + "learning_rate": 7.817671337095244e-07, + "loss": 0.0288, + "reward": -0.3089722655713558, + "reward_after_mean": -0.3089722655713558, + "reward_after_std": 0.3733787778764963, + "reward_before_mean": 0.0013715587556362152, + "reward_before_std": 0.30914535373449326, + "reward_change_max": 0.0, + "reward_change_mean": -0.31034383550286293, + "reward_change_min": -0.46327832341194153, + "reward_change_std": 0.17456937301903963, + "reward_std": 0.3733787890523672, + "rewards/accuracy_reward": 0.12500000558793545, + "rewards/cosine_scaled_reward": -0.12362844927702099, + "step": 80 + }, + { + "clip_fraction": 0.0, + "completion_length": 3043.270866394043, + "epoch": 0.09257142857142857, + "grad_norm": 0.08238786458969116, + "kl": 5.1021575927734375e-05, + "lambda_div_used": 0.5923707559704781, + "learning_rate": 7.75e-07, + "loss": 0.0152, + "reward": -0.26676796935498714, + "reward_after_mean": -0.26676796935498714, + "reward_after_std": 0.4838532619178295, + "reward_before_mean": 0.01490369625389576, + "reward_before_std": 0.4254928780719638, + "reward_change_max": 0.0, + "reward_change_mean": -0.28167167864739895, + "reward_change_min": -0.4777062237262726, + "reward_change_std": 0.17371418420225382, + "reward_std": 0.4838532619178295, + "rewards/accuracy_reward": 0.12500000186264515, + "rewards/cosine_scaled_reward": -0.11009630188345909, + "step": 81 + }, + { + "clip_fraction": 0.0, + "completion_length": 2630.625030517578, + "epoch": 0.09371428571428571, + "grad_norm": 0.06656210124492645, + "kl": 4.544854164123535e-05, + "lambda_div_used": 0.6220841184258461, + "learning_rate": 7.681643291108517e-07, + "loss": 0.0658, + "reward": -0.2009956305846572, + "reward_after_mean": -0.2009956305846572, + "reward_after_std": 0.6119217481464148, + "reward_before_mean": 0.05983926076442003, + "reward_before_std": 0.5658847019076347, + "reward_change_max": 0.0, + "reward_change_mean": -0.2608348857611418, + "reward_change_min": -0.45365019887685776, + "reward_change_std": 0.16699018515646458, + "reward_std": 0.6119217481464148, + "rewards/accuracy_reward": 0.1458333358168602, + "rewards/cosine_scaled_reward": -0.08599409856833518, + "step": 82 + }, + { + "clip_fraction": 0.0, + "completion_length": 2535.208381652832, + "epoch": 0.09485714285714286, + "grad_norm": 0.09338229149580002, + "kl": 5.167722702026367e-05, + "lambda_div_used": 0.5914888307452202, + "learning_rate": 7.612622032536507e-07, + "loss": 0.0644, + "reward": -0.14509151689708233, + "reward_after_mean": -0.14509151689708233, + "reward_after_std": 0.47357647120952606, + "reward_before_mean": 0.1998078590258956, + "reward_before_std": 0.4206119291484356, + "reward_change_max": 0.0, + "reward_change_mean": -0.3448993805795908, + "reward_change_min": -0.5211166813969612, + "reward_change_std": 0.20530468598008156, + "reward_std": 0.4735764730721712, + "rewards/accuracy_reward": 0.18750000558793545, + "rewards/cosine_scaled_reward": 0.012307855300605297, + "step": 83 + }, + { + "clip_fraction": 0.0, + "completion_length": 2915.8750076293945, + "epoch": 0.096, + "grad_norm": 0.07025005668401718, + "kl": 4.2825937271118164e-05, + "lambda_div_used": 0.6353590413928032, + "learning_rate": 7.54295724882796e-07, + "loss": -0.0466, + "reward": -0.1503741154447198, + "reward_after_mean": -0.1503741154447198, + "reward_after_std": 0.6686306204646826, + "reward_before_mean": 0.10233869170770049, + "reward_before_std": 0.6240420090034604, + "reward_change_max": 0.0, + "reward_change_mean": -0.25271278992295265, + "reward_change_min": -0.43359045311808586, + "reward_change_std": 0.1594225950539112, + "reward_std": 0.6686306446790695, + "rewards/accuracy_reward": 0.20833334140479565, + "rewards/cosine_scaled_reward": -0.10599466459825635, + "step": 84 + }, + { + "clip_fraction": 0.0, + "completion_length": 2968.9375610351562, + "epoch": 0.09714285714285714, + "grad_norm": 0.05030859634280205, + "kl": 2.9653310775756836e-05, + "lambda_div_used": 0.6120947226881981, + "learning_rate": 7.472670160550848e-07, + "loss": 0.02, + "reward": -0.31006562570109963, + "reward_after_mean": -0.31006562570109963, + "reward_after_std": 0.5445140562951565, + "reward_before_mean": -0.0690727960318327, + "reward_before_std": 0.5210688021034002, + "reward_change_max": 0.0, + "reward_change_mean": -0.2409928422421217, + "reward_change_min": -0.46906592324376106, + "reward_change_std": 0.1683354014530778, + "reward_std": 0.5445140581578016, + "rewards/accuracy_reward": 0.10416666977107525, + "rewards/cosine_scaled_reward": -0.17323945881798863, + "step": 85 + }, + { + "clip_fraction": 0.0, + "completion_length": 2726.9167098999023, + "epoch": 0.09828571428571428, + "grad_norm": 0.08248579502105713, + "kl": 5.6862831115722656e-05, + "lambda_div_used": 0.5853299722075462, + "learning_rate": 7.401782177833147e-07, + "loss": -0.015, + "reward": -0.32714659720659256, + "reward_after_mean": -0.32714659720659256, + "reward_after_std": 0.4505470525473356, + "reward_before_mean": -0.05685961013659835, + "reward_before_std": 0.39618763769976795, + "reward_change_max": 0.0, + "reward_change_mean": -0.2702869772911072, + "reward_change_min": -0.49061961844563484, + "reward_change_std": 0.17353465128690004, + "reward_std": 0.4505470544099808, + "rewards/accuracy_reward": 0.0833333358168602, + "rewards/cosine_scaled_reward": -0.14019294315949082, + "step": 86 + }, + { + "clip_fraction": 0.0, + "completion_length": 2504.604232788086, + "epoch": 0.09942857142857142, + "grad_norm": 0.07946456968784332, + "kl": 5.137920379638672e-05, + "lambda_div_used": 0.5860441103577614, + "learning_rate": 7.330314893841101e-07, + "loss": -0.0658, + "reward": -0.25398197025060654, + "reward_after_mean": -0.25398197025060654, + "reward_after_std": 0.4263457953929901, + "reward_before_mean": 0.039198137819767, + "reward_before_std": 0.4029952948912978, + "reward_change_max": 0.0, + "reward_change_mean": -0.2931801360100508, + "reward_change_min": -0.46910280734300613, + "reward_change_std": 0.18813357036560774, + "reward_std": 0.4263457991182804, + "rewards/accuracy_reward": 0.1458333358168602, + "rewards/cosine_scaled_reward": -0.1066351905465126, + "step": 87 + }, + { + "clip_fraction": 0.0, + "completion_length": 2021.3958702087402, + "epoch": 0.10057142857142858, + "grad_norm": 0.08914105594158173, + "kl": 3.445148468017578e-05, + "lambda_div_used": 0.6275195479393005, + "learning_rate": 7.258290078201731e-07, + "loss": 0.0661, + "reward": -0.03901347843930125, + "reward_after_mean": -0.03901347843930125, + "reward_after_std": 0.6304188724607229, + "reward_before_mean": 0.28059265296906233, + "reward_before_std": 0.5838784109801054, + "reward_change_max": 0.0, + "reward_change_mean": -0.3196061383932829, + "reward_change_min": -0.4850176088511944, + "reward_change_std": 0.19270309899002314, + "reward_std": 0.6304188761860132, + "rewards/accuracy_reward": 0.25000000931322575, + "rewards/cosine_scaled_reward": 0.030592639930546284, + "step": 88 + }, + { + "clip_fraction": 0.0, + "completion_length": 2855.666702270508, + "epoch": 0.10171428571428572, + "grad_norm": 0.052225928753614426, + "kl": 3.810226917266846e-05, + "lambda_div_used": 0.603381521999836, + "learning_rate": 7.185729670371604e-07, + "loss": -0.0748, + "reward": -0.2931769546121359, + "reward_after_mean": -0.2931769546121359, + "reward_after_std": 0.5186727736145258, + "reward_before_mean": -0.04963091528043151, + "reward_before_std": 0.4795092437416315, + "reward_change_max": 0.0, + "reward_change_mean": -0.2435460388660431, + "reward_change_min": -0.4475689232349396, + "reward_change_std": 0.1609211042523384, + "reward_std": 0.5186727829277515, + "rewards/accuracy_reward": 0.12500000186264515, + "rewards/cosine_scaled_reward": -0.17463092133402824, + "step": 89 + }, + { + "clip_fraction": 0.0, + "completion_length": 2377.9375228881836, + "epoch": 0.10285714285714286, + "grad_norm": 0.1115579754114151, + "kl": 6.511807441711426e-05, + "lambda_div_used": 0.5806571692228317, + "learning_rate": 7.11265577295385e-07, + "loss": 0.1049, + "reward": -0.39063363266177475, + "reward_after_mean": -0.39063363266177475, + "reward_after_std": 0.42313366010785103, + "reward_before_mean": -0.14117828011512756, + "reward_before_std": 0.3631622865796089, + "reward_change_max": 0.0, + "reward_change_mean": -0.24945535697042942, + "reward_change_min": -0.38458868488669395, + "reward_change_std": 0.14130910206586123, + "reward_std": 0.4231336638331413, + "rewards/accuracy_reward": 0.0416666679084301, + "rewards/cosine_scaled_reward": -0.1828449461609125, + "step": 90 + }, + { + "clip_fraction": 0.0, + "completion_length": 2758.250015258789, + "epoch": 0.104, + "grad_norm": 0.0690622553229332, + "kl": 4.595518112182617e-05, + "lambda_div_used": 0.5834402665495872, + "learning_rate": 7.039090644965509e-07, + "loss": 0.0142, + "reward": -0.1535566644743085, + "reward_after_mean": -0.1535566644743085, + "reward_after_std": 0.4708296097815037, + "reward_before_mean": 0.22540673054754734, + "reward_before_std": 0.3833985608071089, + "reward_change_max": 0.0, + "reward_change_mean": -0.3789634220302105, + "reward_change_min": -0.6174517869949341, + "reward_change_std": 0.2261828240007162, + "reward_std": 0.4708296228200197, + "rewards/accuracy_reward": 0.2291666679084301, + "rewards/cosine_scaled_reward": -0.003759911749511957, + "step": 91 + }, + { + "clip_fraction": 0.0, + "completion_length": 2467.2083892822266, + "epoch": 0.10514285714285715, + "grad_norm": 0.07518215477466583, + "kl": 3.6522746086120605e-05, + "lambda_div_used": 0.5778974890708923, + "learning_rate": 6.965056695057204e-07, + "loss": 0.0134, + "reward": -0.324931837618351, + "reward_after_mean": -0.324931837618351, + "reward_after_std": 0.42975695990025997, + "reward_before_mean": -0.03748153988271952, + "reward_before_std": 0.35468481201678514, + "reward_change_max": 0.0, + "reward_change_mean": -0.2874503042548895, + "reward_change_min": -0.4382200054824352, + "reward_change_std": 0.1673103515058756, + "reward_std": 0.42975698225200176, + "rewards/accuracy_reward": 0.1041666679084301, + "rewards/cosine_scaled_reward": -0.14164821058511734, + "step": 92 + }, + { + "clip_fraction": 0.0, + "completion_length": 3584.0, + "epoch": 0.10628571428571429, + "grad_norm": 0.05293423309922218, + "kl": 5.128979682922363e-05, + "lambda_div_used": 0.5581478402018547, + "learning_rate": 6.890576474687263e-07, + "loss": -0.0, + "reward": -0.4109080731868744, + "reward_after_mean": -0.4109080731868744, + "reward_after_std": 0.33292335644364357, + "reward_before_mean": -0.12180843483656645, + "reward_before_std": 0.26414576172828674, + "reward_change_max": 0.0, + "reward_change_mean": -0.28909964114427567, + "reward_change_min": -0.45860791578888893, + "reward_change_std": 0.16918409056961536, + "reward_std": 0.332923362031579, + "rewards/accuracy_reward": 0.0416666679084301, + "rewards/cosine_scaled_reward": -0.16347510018385947, + "step": 93 + }, + { + "clip_fraction": 0.0, + "completion_length": 2337.0625076293945, + "epoch": 0.10742857142857143, + "grad_norm": 0.08120116591453552, + "kl": 4.356354475021362e-05, + "lambda_div_used": 0.5623572915792465, + "learning_rate": 6.815672671252315e-07, + "loss": 0.0672, + "reward": -0.2148991823196411, + "reward_after_mean": -0.2148991823196411, + "reward_after_std": 0.37764647975564003, + "reward_before_mean": 0.17423074319958687, + "reward_before_std": 0.2815079055726528, + "reward_change_max": 0.0, + "reward_change_mean": -0.3891299031674862, + "reward_change_min": -0.5794133953750134, + "reward_change_std": 0.21886237617582083, + "reward_std": 0.3776464983820915, + "rewards/accuracy_reward": 0.2083333358168602, + "rewards/cosine_scaled_reward": -0.034102603793144226, + "step": 94 + }, + { + "clip_fraction": 0.0, + "completion_length": 3268.7708587646484, + "epoch": 0.10857142857142857, + "grad_norm": 0.048412173986434937, + "kl": 3.3482909202575684e-05, + "lambda_div_used": 0.599761851131916, + "learning_rate": 6.740368101176495e-07, + "loss": 0.0138, + "reward": -0.28226344753056765, + "reward_after_mean": -0.28226344753056765, + "reward_after_std": 0.5015552807599306, + "reward_before_mean": -0.01817359635606408, + "reward_before_std": 0.4594444427639246, + "reward_change_max": 0.0, + "reward_change_mean": -0.2640898581594229, + "reward_change_min": -0.45775213465094566, + "reward_change_std": 0.170160255394876, + "reward_std": 0.5015552863478661, + "rewards/accuracy_reward": 0.12500000186264515, + "rewards/cosine_scaled_reward": -0.14317359682172537, + "step": 95 + }, + { + "clip_fraction": 0.0, + "completion_length": 2311.229202270508, + "epoch": 0.10971428571428571, + "grad_norm": 0.08286502212285995, + "kl": 3.9830803871154785e-05, + "lambda_div_used": 0.6152824014425278, + "learning_rate": 6.664685702961344e-07, + "loss": 0.0394, + "reward": -0.1798182651400566, + "reward_after_mean": -0.1798182651400566, + "reward_after_std": 0.5684184953570366, + "reward_before_mean": 0.09630595671478659, + "reward_before_std": 0.5319525888189673, + "reward_change_max": 0.0, + "reward_change_mean": -0.2761242166161537, + "reward_change_min": -0.4500977620482445, + "reward_change_std": 0.17272682767361403, + "reward_std": 0.5684185232967138, + "rewards/accuracy_reward": 0.2083333395421505, + "rewards/cosine_scaled_reward": -0.11202737595885992, + "step": 96 + }, + { + "clip_fraction": 0.0, + "completion_length": 3088.104202270508, + "epoch": 0.11085714285714286, + "grad_norm": 0.05773423984646797, + "kl": 4.525482654571533e-05, + "lambda_div_used": 0.5830484703183174, + "learning_rate": 6.588648530198504e-07, + "loss": 0.012, + "reward": -0.24631217122077942, + "reward_after_mean": -0.24631217122077942, + "reward_after_std": 0.4177880808711052, + "reward_before_mean": 0.052551381289958954, + "reward_before_std": 0.3777043428272009, + "reward_change_max": 0.0, + "reward_change_mean": -0.2988635450601578, + "reward_change_min": -0.4822593107819557, + "reward_change_std": 0.18344944156706333, + "reward_std": 0.4177880957722664, + "rewards/accuracy_reward": 0.1666666716337204, + "rewards/cosine_scaled_reward": -0.11411529779434204, + "step": 97 + }, + { + "clip_fraction": 0.0, + "completion_length": 3099.6666870117188, + "epoch": 0.112, + "grad_norm": 0.05776653811335564, + "kl": 4.242360591888428e-05, + "lambda_div_used": 0.5548205152153969, + "learning_rate": 6.512279744547392e-07, + "loss": 0.0853, + "reward": -0.30739316577091813, + "reward_after_mean": -0.30739316577091813, + "reward_after_std": 0.3838734310120344, + "reward_before_mean": 0.044590696692466736, + "reward_before_std": 0.2453754236921668, + "reward_change_max": 0.0, + "reward_change_mean": -0.3519838694483042, + "reward_change_min": -0.49566248431801796, + "reward_change_std": 0.18073826655745506, + "reward_std": 0.38387343287467957, + "rewards/accuracy_reward": 0.14583333395421505, + "rewards/cosine_scaled_reward": -0.10124263912439346, + "step": 98 + }, + { + "clip_fraction": 0.0, + "completion_length": 2814.0416679382324, + "epoch": 0.11314285714285714, + "grad_norm": 0.10099756717681885, + "kl": 3.9711594581604004e-05, + "lambda_div_used": 0.589689776301384, + "learning_rate": 6.435602608679916e-07, + "loss": -0.0116, + "reward": -0.24916072934865952, + "reward_after_mean": -0.24916072934865952, + "reward_after_std": 0.5363867282867432, + "reward_before_mean": 0.07555947656510398, + "reward_before_std": 0.4080575914122164, + "reward_change_max": 0.0, + "reward_change_mean": -0.32472023367881775, + "reward_change_min": -0.46621018648147583, + "reward_change_std": 0.16967851482331753, + "reward_std": 0.5363867320120335, + "rewards/accuracy_reward": 0.18750000186264515, + "rewards/cosine_scaled_reward": -0.11194050963968039, + "step": 99 + }, + { + "clip_fraction": 0.0, + "completion_length": 2585.208366394043, + "epoch": 0.11428571428571428, + "grad_norm": 0.08013861626386642, + "kl": 3.557652235031128e-05, + "lambda_div_used": 0.6440299674868584, + "learning_rate": 6.358640479194451e-07, + "loss": 0.0233, + "reward": -0.017059004865586758, + "reward_after_mean": -0.017059004865586758, + "reward_after_std": 0.6788044832646847, + "reward_before_mean": 0.27609538938850164, + "reward_before_std": 0.672226045280695, + "reward_change_max": 0.0, + "reward_change_mean": -0.29315441474318504, + "reward_change_min": -0.560445386916399, + "reward_change_std": 0.2121435971930623, + "reward_std": 0.6788045018911362, + "rewards/accuracy_reward": 0.25000000186264515, + "rewards/cosine_scaled_reward": 0.02609538659453392, + "step": 100 + }, + { + "clip_fraction": 0.0, + "completion_length": 2698.687515258789, + "epoch": 0.11542857142857142, + "grad_norm": 0.08196399360895157, + "kl": 4.9740076065063477e-05, + "lambda_div_used": 0.5603137612342834, + "learning_rate": 6.281416799501187e-07, + "loss": -0.0552, + "reward": -0.2113272361457348, + "reward_after_mean": -0.2113272361457348, + "reward_after_std": 0.3802106771618128, + "reward_before_mean": 0.176470085978508, + "reward_before_std": 0.27200379874557257, + "reward_change_max": 0.0, + "reward_change_mean": -0.3877973351627588, + "reward_change_min": -0.5599614717066288, + "reward_change_std": 0.21361587569117546, + "reward_std": 0.38021068647503853, + "rewards/accuracy_reward": 0.2083333358168602, + "rewards/cosine_scaled_reward": -0.03186323493719101, + "step": 101 + }, + { + "clip_fraction": 0.0, + "completion_length": 2008.1875457763672, + "epoch": 0.11657142857142858, + "grad_norm": 0.09427309036254883, + "kl": 3.542006015777588e-05, + "lambda_div_used": 0.5826256647706032, + "learning_rate": 6.203955092681039e-07, + "loss": -0.0045, + "reward": -0.2004177300259471, + "reward_after_mean": -0.2004177300259471, + "reward_after_std": 0.498240664601326, + "reward_before_mean": 0.16466995794326067, + "reward_before_std": 0.37351171765476465, + "reward_change_max": 0.0, + "reward_change_mean": -0.36508770659565926, + "reward_change_min": -0.5195483043789864, + "reward_change_std": 0.19531819131225348, + "reward_std": 0.4982406720519066, + "rewards/accuracy_reward": 0.1666666679084301, + "rewards/cosine_scaled_reward": -0.001996707171201706, + "step": 102 + }, + { + "clip_fraction": 0.0, + "completion_length": 2895.250020980835, + "epoch": 0.11771428571428572, + "grad_norm": 0.06802447140216827, + "kl": 4.501640796661377e-05, + "lambda_div_used": 0.5506640374660492, + "learning_rate": 6.126278954320294e-07, + "loss": 0.0391, + "reward": -0.3214366286993027, + "reward_after_mean": -0.3214366286993027, + "reward_after_std": 0.3532369527965784, + "reward_before_mean": 0.050119780004024506, + "reward_before_std": 0.22584082814864814, + "reward_change_max": 0.0, + "reward_change_mean": -0.3715564049780369, + "reward_change_min": -0.520975548774004, + "reward_change_std": 0.19540261384099722, + "reward_std": 0.35323695838451385, + "rewards/accuracy_reward": 0.14583333395421505, + "rewards/cosine_scaled_reward": -0.09571357164531946, + "step": 103 + }, + { + "clip_fraction": 0.0, + "completion_length": 2521.3333435058594, + "epoch": 0.11885714285714286, + "grad_norm": 0.096355140209198, + "kl": 4.947185516357422e-05, + "lambda_div_used": 0.5742901861667633, + "learning_rate": 6.048412045323164e-07, + "loss": 0.0001, + "reward": -0.32155255414545536, + "reward_after_mean": -0.32155255414545536, + "reward_after_std": 0.39902236871421337, + "reward_before_mean": -0.03546402044594288, + "reward_before_std": 0.3418318200856447, + "reward_change_max": 0.0, + "reward_change_mean": -0.2860885336995125, + "reward_change_min": -0.4318212755024433, + "reward_change_std": 0.16667354479432106, + "reward_std": 0.39902237243950367, + "rewards/accuracy_reward": 0.10416666977107525, + "rewards/cosine_scaled_reward": -0.13963068462908268, + "step": 104 + }, + { + "clip_fraction": 0.0, + "completion_length": 2526.000030517578, + "epoch": 0.12, + "grad_norm": 0.07842111587524414, + "kl": 4.8801302909851074e-05, + "lambda_div_used": 0.6022165417671204, + "learning_rate": 5.97037808470444e-07, + "loss": 0.0569, + "reward": -0.12751667387783527, + "reward_after_mean": -0.12751667387783527, + "reward_after_std": 0.582199590280652, + "reward_before_mean": 0.24656505044549704, + "reward_before_std": 0.4756303254980594, + "reward_change_max": 0.0, + "reward_change_mean": -0.37408171594142914, + "reward_change_min": -0.5777854695916176, + "reward_change_std": 0.22329542227089405, + "reward_std": 0.5821996051818132, + "rewards/accuracy_reward": 0.25000000186264515, + "rewards/cosine_scaled_reward": -0.0034349607303738594, + "step": 105 + }, + { + "clip_fraction": 0.0, + "completion_length": 2396.291702270508, + "epoch": 0.12114285714285715, + "grad_norm": 0.07387516647577286, + "kl": 2.6203226298093796e-05, + "lambda_div_used": 0.6128493994474411, + "learning_rate": 5.892200842364462e-07, + "loss": -0.0293, + "reward": 0.159407502040267, + "reward_after_mean": 0.159407502040267, + "reward_after_std": 0.5759131647646427, + "reward_before_mean": 0.6274341251701117, + "reward_before_std": 0.5208892030641437, + "reward_change_max": 0.0, + "reward_change_mean": -0.46802657656371593, + "reward_change_min": -0.7112628631293774, + "reward_change_std": 0.2871347274631262, + "reward_std": 0.5759131908416748, + "rewards/accuracy_reward": 0.4583333469927311, + "rewards/cosine_scaled_reward": 0.16910075349733233, + "step": 106 + }, + { + "clip_fraction": 0.0, + "completion_length": 2806.875045776367, + "epoch": 0.12228571428571429, + "grad_norm": 0.08797896653413773, + "kl": 4.5686960220336914e-05, + "lambda_div_used": 0.5551519840955734, + "learning_rate": 5.813904131848564e-07, + "loss": -0.0206, + "reward": -0.20906101167201996, + "reward_after_mean": -0.20906101167201996, + "reward_after_std": 0.35940215550363064, + "reward_before_mean": 0.19015613943338394, + "reward_before_std": 0.24773720651865005, + "reward_change_max": 0.0, + "reward_change_mean": -0.39921717159450054, + "reward_change_min": -0.5645803362131119, + "reward_change_std": 0.21728475205600262, + "reward_std": 0.35940215922892094, + "rewards/accuracy_reward": 0.2291666716337204, + "rewards/cosine_scaled_reward": -0.03901051543653011, + "step": 107 + }, + { + "clip_fraction": 0.0, + "completion_length": 2676.3125381469727, + "epoch": 0.12342857142857143, + "grad_norm": 0.08117339015007019, + "kl": 4.683062434196472e-05, + "lambda_div_used": 0.5782058760523796, + "learning_rate": 5.735511803093248e-07, + "loss": 0.0222, + "reward": -0.33698799335979857, + "reward_after_mean": -0.33698799335979857, + "reward_after_std": 0.4150846730917692, + "reward_before_mean": -0.06898907572031021, + "reward_before_std": 0.3580573983490467, + "reward_change_max": 0.0, + "reward_change_mean": -0.2679989282041788, + "reward_change_min": -0.39731478318572044, + "reward_change_std": 0.15669311955571175, + "reward_std": 0.41508468240499496, + "rewards/accuracy_reward": 0.10416666977107525, + "rewards/cosine_scaled_reward": -0.17315573990345, + "step": 108 + }, + { + "clip_fraction": 0.0, + "completion_length": 3091.4166870117188, + "epoch": 0.12457142857142857, + "grad_norm": 0.06771776080131531, + "kl": 4.081428050994873e-05, + "lambda_div_used": 0.5580320879817009, + "learning_rate": 5.657047735161255e-07, + "loss": -0.0162, + "reward": -0.30369802191853523, + "reward_after_mean": -0.30369802191853523, + "reward_after_std": 0.3783009462058544, + "reward_before_mean": 0.060508192516863346, + "reward_before_std": 0.2597539462149143, + "reward_change_max": 0.0, + "reward_change_mean": -0.36420623771846294, + "reward_change_min": -0.5151935815811157, + "reward_change_std": 0.19158014561980963, + "reward_std": 0.37830095551908016, + "rewards/accuracy_reward": 0.14583333395421505, + "rewards/cosine_scaled_reward": -0.08532514423131943, + "step": 109 + }, + { + "clip_fraction": 0.0, + "completion_length": 2727.875045776367, + "epoch": 0.12571428571428572, + "grad_norm": 0.07848876714706421, + "kl": 3.8996338844299316e-05, + "lambda_div_used": 0.5898331105709076, + "learning_rate": 5.578535828967777e-07, + "loss": 0.0059, + "reward": -0.27789773186668754, + "reward_after_mean": -0.27789773186668754, + "reward_after_std": 0.48964778520166874, + "reward_before_mean": 0.006700331810861826, + "reward_before_std": 0.40999170672148466, + "reward_change_max": 0.0, + "reward_change_mean": -0.28459805622696877, + "reward_change_min": -0.4541938379406929, + "reward_change_std": 0.16897330060601234, + "reward_std": 0.48964780382812023, + "rewards/accuracy_reward": 0.16666667349636555, + "rewards/cosine_scaled_reward": -0.15996634401381016, + "step": 110 + }, + { + "clip_fraction": 0.0, + "completion_length": 3152.3958587646484, + "epoch": 0.12685714285714286, + "grad_norm": 0.06026454642415047, + "kl": 5.206465721130371e-05, + "lambda_div_used": 0.6035187616944313, + "learning_rate": 5.5e-07, + "loss": 0.0004, + "reward": -0.22968922927975655, + "reward_after_mean": -0.22968922927975655, + "reward_after_std": 0.5089491438120604, + "reward_before_mean": 0.050578076392412186, + "reward_before_std": 0.47721442952752113, + "reward_change_max": 0.0, + "reward_change_mean": -0.28026728704571724, + "reward_change_min": -0.4855511710047722, + "reward_change_std": 0.1824399819597602, + "reward_std": 0.5089491568505764, + "rewards/accuracy_reward": 0.14583333395421505, + "rewards/cosine_scaled_reward": -0.09525526221841574, + "step": 111 + }, + { + "clip_fraction": 0.0, + "completion_length": 3253.937530517578, + "epoch": 0.128, + "grad_norm": 0.054045893251895905, + "kl": 4.832446575164795e-05, + "lambda_div_used": 0.6144762486219406, + "learning_rate": 5.421464171032224e-07, + "loss": 0.0014, + "reward": -0.07707784557715058, + "reward_after_mean": -0.07707784557715058, + "reward_after_std": 0.5871799997985363, + "reward_before_mean": 0.2822006791830063, + "reward_before_std": 0.5333117246627808, + "reward_change_max": 0.0, + "reward_change_mean": -0.35927851870656013, + "reward_change_min": -0.6481843888759613, + "reward_change_std": 0.23641589283943176, + "reward_std": 0.5871800072491169, + "rewards/accuracy_reward": 0.2708333358168602, + "rewards/cosine_scaled_reward": 0.01136732567101717, + "step": 112 + }, + { + "clip_fraction": 0.0, + "completion_length": 2541.812545776367, + "epoch": 0.12914285714285714, + "grad_norm": 0.09499017894268036, + "kl": 4.720315337181091e-05, + "lambda_div_used": 0.5564066097140312, + "learning_rate": 5.342952264838747e-07, + "loss": 0.0471, + "reward": -0.3626333475112915, + "reward_after_mean": -0.3626333475112915, + "reward_after_std": 0.3075137473642826, + "reward_before_mean": -0.05370184499770403, + "reward_before_std": 0.25310691073536873, + "reward_change_max": 0.0, + "reward_change_mean": -0.30893150344491005, + "reward_change_min": -0.4691920429468155, + "reward_change_std": 0.1770205283537507, + "reward_std": 0.3075137585401535, + "rewards/accuracy_reward": 0.0833333358168602, + "rewards/cosine_scaled_reward": -0.13703518453985453, + "step": 113 + }, + { + "clip_fraction": 0.0, + "completion_length": 2547.5416984558105, + "epoch": 0.13028571428571428, + "grad_norm": 0.06982935965061188, + "kl": 2.650544047355652e-05, + "lambda_div_used": 0.5948992818593979, + "learning_rate": 5.264488196906752e-07, + "loss": 0.0506, + "reward": -0.33033538423478603, + "reward_after_mean": -0.33033538423478603, + "reward_after_std": 0.49076898768544197, + "reward_before_mean": -0.08067071554251015, + "reward_before_std": 0.4387869122438133, + "reward_change_max": 0.0, + "reward_change_mean": -0.2496646661311388, + "reward_change_min": -0.38466524705290794, + "reward_change_std": 0.15147447120398283, + "reward_std": 0.4907689895480871, + "rewards/accuracy_reward": 0.1041666679084301, + "rewards/cosine_scaled_reward": -0.18483738787472248, + "step": 114 + }, + { + "clip_fraction": 0.0, + "completion_length": 2926.3125228881836, + "epoch": 0.13142857142857142, + "grad_norm": 0.10055308789014816, + "kl": 4.2185187339782715e-05, + "lambda_div_used": 0.5866656303405762, + "learning_rate": 5.186095868151436e-07, + "loss": 0.0219, + "reward": -0.16912975907325745, + "reward_after_mean": -0.16912975907325745, + "reward_after_std": 0.4218728318810463, + "reward_before_mean": 0.17230269685387611, + "reward_before_std": 0.39482294395565987, + "reward_change_max": 0.0, + "reward_change_mean": -0.34143244847655296, + "reward_change_min": -0.5322228632867336, + "reward_change_std": 0.21214590221643448, + "reward_std": 0.4218728430569172, + "rewards/accuracy_reward": 0.2708333432674408, + "rewards/cosine_scaled_reward": -0.09853065386414528, + "step": 115 + }, + { + "clip_fraction": 0.0, + "completion_length": 3185.895835876465, + "epoch": 0.13257142857142856, + "grad_norm": 0.06827546656131744, + "kl": 4.4345855712890625e-05, + "lambda_div_used": 0.579861506819725, + "learning_rate": 5.107799157635538e-07, + "loss": -0.017, + "reward": -0.36347829084843397, + "reward_after_mean": -0.36347829084843397, + "reward_after_std": 0.425207169726491, + "reward_before_mean": -0.10740169882774353, + "reward_before_std": 0.36591998394578695, + "reward_change_max": 0.0, + "reward_change_mean": -0.25607660599052906, + "reward_change_min": -0.4011174105107784, + "reward_change_std": 0.15214112866669893, + "reward_std": 0.4252071734517813, + "rewards/accuracy_reward": 0.08333333395421505, + "rewards/cosine_scaled_reward": -0.19073502300307155, + "step": 116 + }, + { + "clip_fraction": 0.0, + "completion_length": 3265.666702270508, + "epoch": 0.1337142857142857, + "grad_norm": 0.058839015662670135, + "kl": 5.224347114562988e-05, + "lambda_div_used": 0.5589891448616982, + "learning_rate": 5.02962191529556e-07, + "loss": 0.0144, + "reward": -0.47565145045518875, + "reward_after_mean": -0.47565145045518875, + "reward_after_std": 0.34672038443386555, + "reward_before_mean": -0.2341720014810562, + "reward_before_std": 0.2644944768399, + "reward_change_max": 0.0, + "reward_change_mean": -0.2414794433861971, + "reward_change_min": -0.3730311393737793, + "reward_change_std": 0.13014927878975868, + "reward_std": 0.3467204011976719, + "rewards/accuracy_reward": 0.02083333395421505, + "rewards/cosine_scaled_reward": -0.2550053410232067, + "step": 117 + }, + { + "clip_fraction": 0.0, + "completion_length": 3087.9375610351562, + "epoch": 0.13485714285714287, + "grad_norm": 0.05542779713869095, + "kl": 3.71783971786499e-05, + "lambda_div_used": 0.6410617902874947, + "learning_rate": 4.951587954676837e-07, + "loss": 0.0602, + "reward": 0.02640039217658341, + "reward_after_mean": 0.02640039217658341, + "reward_after_std": 0.6670121420174837, + "reward_before_mean": 0.34903212962672114, + "reward_before_std": 0.6545964349061251, + "reward_change_max": 0.0, + "reward_change_mean": -0.322631748393178, + "reward_change_min": -0.5322613082826138, + "reward_change_std": 0.21465477347373962, + "reward_std": 0.66701215878129, + "rewards/accuracy_reward": 0.2708333395421505, + "rewards/cosine_scaled_reward": 0.07819879939779639, + "step": 118 + }, + { + "clip_fraction": 0.0, + "completion_length": 2036.8542251586914, + "epoch": 0.136, + "grad_norm": 0.09512177854776382, + "kl": 4.920363426208496e-05, + "lambda_div_used": 0.5765073597431183, + "learning_rate": 4.873721045679706e-07, + "loss": 0.0968, + "reward": -0.1327025555074215, + "reward_after_mean": -0.1327025555074215, + "reward_after_std": 0.4383635278791189, + "reward_before_mean": 0.2736157886683941, + "reward_before_std": 0.34712607227265835, + "reward_change_max": 0.0, + "reward_change_mean": -0.4063183292746544, + "reward_change_min": -0.5770336836576462, + "reward_change_std": 0.22668993193656206, + "reward_std": 0.43836353346705437, + "rewards/accuracy_reward": 0.22916666977107525, + "rewards/cosine_scaled_reward": 0.0444490984082222, + "step": 119 + }, + { + "clip_fraction": 0.0, + "completion_length": 2693.6458854675293, + "epoch": 0.13714285714285715, + "grad_norm": 0.0801864042878151, + "kl": 5.1856040954589844e-05, + "lambda_div_used": 0.5954036563634872, + "learning_rate": 4.79604490731896e-07, + "loss": -0.0416, + "reward": -0.10674675926566124, + "reward_after_mean": -0.10674675926566124, + "reward_after_std": 0.5345237273722887, + "reward_before_mean": 0.26713848020881414, + "reward_before_std": 0.44027570402249694, + "reward_change_max": 0.0, + "reward_change_mean": -0.37388524040579796, + "reward_change_min": -0.5998699106276035, + "reward_change_std": 0.22230207175016403, + "reward_std": 0.5345237515866756, + "rewards/accuracy_reward": 0.25000000186264515, + "rewards/cosine_scaled_reward": 0.017138468101620674, + "step": 120 + }, + { + "clip_fraction": 0.0, + "completion_length": 1960.8958587646484, + "epoch": 0.1382857142857143, + "grad_norm": 0.09297072887420654, + "kl": 3.6925775930285454e-05, + "lambda_div_used": 0.5758452340960503, + "learning_rate": 4.7185832004988133e-07, + "loss": -0.0275, + "reward": -0.15540813654661179, + "reward_after_mean": -0.15540813654661179, + "reward_after_std": 0.3896722886711359, + "reward_before_mean": 0.20818629674613476, + "reward_before_std": 0.34600438084453344, + "reward_change_max": 0.0, + "reward_change_mean": -0.3635944165289402, + "reward_change_min": -0.5350156500935555, + "reward_change_std": 0.21344942972064018, + "reward_std": 0.38967230543494225, + "rewards/accuracy_reward": 0.1875000074505806, + "rewards/cosine_scaled_reward": 0.020686281844973564, + "step": 121 + }, + { + "clip_fraction": 0.0, + "completion_length": 2996.7917289733887, + "epoch": 0.13942857142857143, + "grad_norm": 0.07053450495004654, + "kl": 5.537271499633789e-05, + "lambda_div_used": 0.604009747505188, + "learning_rate": 4.641359520805548e-07, + "loss": 0.0244, + "reward": -0.18336665583774447, + "reward_after_mean": -0.18336665583774447, + "reward_after_std": 0.5550395585596561, + "reward_before_mean": 0.11408489104360342, + "reward_before_std": 0.4755072835832834, + "reward_change_max": 0.0, + "reward_change_mean": -0.29745154455304146, + "reward_change_min": -0.4127044528722763, + "reward_change_std": 0.16168679296970367, + "reward_std": 0.5550395771861076, + "rewards/accuracy_reward": 0.16666667349636555, + "rewards/cosine_scaled_reward": -0.05258178571239114, + "step": 122 + }, + { + "clip_fraction": 0.0, + "completion_length": 2965.687530517578, + "epoch": 0.14057142857142857, + "grad_norm": 0.061138082295656204, + "kl": 4.2922794818878174e-05, + "lambda_div_used": 0.5864768177270889, + "learning_rate": 4.5643973913200837e-07, + "loss": 0.0629, + "reward": -0.24433407932519913, + "reward_after_mean": -0.24433407932519913, + "reward_after_std": 0.43522679433226585, + "reward_before_mean": 0.05285754054784775, + "reward_before_std": 0.4003843404352665, + "reward_change_max": 0.0, + "reward_change_mean": -0.2971916198730469, + "reward_change_min": -0.4781832844018936, + "reward_change_std": 0.18817945942282677, + "reward_std": 0.435226796194911, + "rewards/accuracy_reward": 0.1458333358168602, + "rewards/cosine_scaled_reward": -0.09297579154372215, + "step": 123 + }, + { + "clip_fraction": 0.0, + "completion_length": 2273.895881652832, + "epoch": 0.1417142857142857, + "grad_norm": 0.06792795658111572, + "kl": 2.843700349330902e-05, + "lambda_div_used": 0.5992666333913803, + "learning_rate": 4.4877202554526084e-07, + "loss": 0.0724, + "reward": -0.030117375776171684, + "reward_after_mean": -0.030117375776171684, + "reward_after_std": 0.6018249355256557, + "reward_before_mean": 0.40752510842867196, + "reward_before_std": 0.4566116305068135, + "reward_change_max": 0.0, + "reward_change_mean": -0.4376424718648195, + "reward_change_min": -0.6283221691846848, + "reward_change_std": 0.2420077919960022, + "reward_std": 0.601824939250946, + "rewards/accuracy_reward": 0.3541666679084301, + "rewards/cosine_scaled_reward": 0.05335840582847595, + "step": 124 + }, + { + "clip_fraction": 0.0, + "completion_length": 2841.687515258789, + "epoch": 0.14285714285714285, + "grad_norm": 0.057179540395736694, + "kl": 3.587547689676285e-05, + "lambda_div_used": 0.5674227699637413, + "learning_rate": 4.4113514698014953e-07, + "loss": 0.0092, + "reward": -0.09743379801511765, + "reward_after_mean": -0.09743379801511765, + "reward_after_std": 0.4501750059425831, + "reward_before_mean": 0.3561172317713499, + "reward_before_std": 0.3082335013896227, + "reward_change_max": 0.0, + "reward_change_mean": -0.453551035374403, + "reward_change_min": -0.6685109585523605, + "reward_change_std": 0.25046134926378727, + "reward_std": 0.4501750282943249, + "rewards/accuracy_reward": 0.3125, + "rewards/cosine_scaled_reward": 0.043617233633995056, + "step": 125 + }, + { + "clip_fraction": 0.0, + "completion_length": 2814.3333892822266, + "epoch": 0.144, + "grad_norm": 0.06912728399038315, + "kl": 3.851950168609619e-05, + "lambda_div_used": 0.6011649072170258, + "learning_rate": 4.3353142970386557e-07, + "loss": 0.0079, + "reward": -0.2701383363455534, + "reward_after_mean": -0.2701383363455534, + "reward_after_std": 0.517305538058281, + "reward_before_mean": -0.00174633227288723, + "reward_before_std": 0.46759936958551407, + "reward_change_max": 0.0, + "reward_change_mean": -0.2683920059353113, + "reward_change_min": -0.42111679539084435, + "reward_change_std": 0.16303524654358625, + "reward_std": 0.5173055771738291, + "rewards/accuracy_reward": 0.1041666679084301, + "rewards/cosine_scaled_reward": -0.10591300774831325, + "step": 126 + }, + { + "clip_fraction": 0.0, + "completion_length": 3520.375, + "epoch": 0.14514285714285713, + "grad_norm": 0.05083320662379265, + "kl": 3.781914710998535e-05, + "lambda_div_used": 0.5308314934372902, + "learning_rate": 4.2596318988235037e-07, + "loss": 0.0094, + "reward": -0.5137354172766209, + "reward_after_mean": -0.5137354172766209, + "reward_after_std": 0.23104721494019032, + "reward_before_mean": -0.23012915067374706, + "reward_before_std": 0.13533379370346665, + "reward_change_max": 0.0, + "reward_change_mean": -0.28360624983906746, + "reward_change_min": -0.4074147306382656, + "reward_change_std": 0.14828919060528278, + "reward_std": 0.2310472223907709, + "rewards/accuracy_reward": 0.0, + "rewards/cosine_scaled_reward": -0.2301291599869728, + "step": 127 + }, + { + "clip_fraction": 0.0, + "completion_length": 2622.6041946411133, + "epoch": 0.1462857142857143, + "grad_norm": 0.07047850638628006, + "kl": 3.9443373680114746e-05, + "lambda_div_used": 0.5977307558059692, + "learning_rate": 4.1843273287476854e-07, + "loss": 0.0245, + "reward": 0.14617427624762058, + "reward_after_mean": 0.14617427624762058, + "reward_after_std": 0.5826170947402716, + "reward_before_mean": 0.6620007424353389, + "reward_before_std": 0.45144926803186536, + "reward_change_max": 0.0, + "reward_change_mean": -0.5158264562487602, + "reward_change_min": -0.7390152402222157, + "reward_change_std": 0.29064416885375977, + "reward_std": 0.5826170966029167, + "rewards/accuracy_reward": 0.41666666977107525, + "rewards/cosine_scaled_reward": 0.2453340534120798, + "step": 128 + }, + { + "clip_fraction": 0.0, + "completion_length": 3211.166702270508, + "epoch": 0.14742857142857144, + "grad_norm": 0.06908967345952988, + "kl": 4.245340824127197e-05, + "lambda_div_used": 0.579024501144886, + "learning_rate": 4.1094235253127374e-07, + "loss": -0.0068, + "reward": -0.41220802813768387, + "reward_after_mean": -0.41220802813768387, + "reward_after_std": 0.4235076569020748, + "reward_before_mean": -0.1792316001956351, + "reward_before_std": 0.3573149349540472, + "reward_change_max": 0.0, + "reward_change_mean": -0.2329764310270548, + "reward_change_min": -0.38812074810266495, + "reward_change_std": 0.13700235076248646, + "reward_std": 0.423507671803236, + "rewards/accuracy_reward": 0.06250000186264515, + "rewards/cosine_scaled_reward": -0.24173159897327423, + "step": 129 + }, + { + "clip_fraction": 0.0, + "completion_length": 3416.375030517578, + "epoch": 0.14857142857142858, + "grad_norm": 0.05305078998208046, + "kl": 3.8370490074157715e-05, + "lambda_div_used": 0.5585792362689972, + "learning_rate": 4.034943304942796e-07, + "loss": -0.0353, + "reward": -0.324653722345829, + "reward_after_mean": -0.324653722345829, + "reward_after_std": 0.30889566242694855, + "reward_before_mean": 0.0034640291705727577, + "reward_before_std": 0.2620142959058285, + "reward_change_max": 0.0, + "reward_change_mean": -0.32811774499714375, + "reward_change_min": -0.4815611355006695, + "reward_change_std": 0.18737321346998215, + "reward_std": 0.30889566615223885, + "rewards/accuracy_reward": 0.1041666716337204, + "rewards/cosine_scaled_reward": -0.10070264618843794, + "step": 130 + }, + { + "clip_fraction": 0.0, + "completion_length": 2672.8541984558105, + "epoch": 0.14971428571428572, + "grad_norm": 0.08398960530757904, + "kl": 2.8876587748527527e-05, + "lambda_div_used": 0.6004800871014595, + "learning_rate": 3.9609093550344907e-07, + "loss": 0.0142, + "reward": 0.04825104773044586, + "reward_after_mean": 0.04825104773044586, + "reward_after_std": 0.537496130913496, + "reward_before_mean": 0.48470486514270306, + "reward_before_std": 0.4661959493532777, + "reward_change_max": 0.0, + "reward_change_mean": -0.43645381927490234, + "reward_change_min": -0.650093249976635, + "reward_change_std": 0.2640222804620862, + "reward_std": 0.5374961327761412, + "rewards/accuracy_reward": 0.3750000074505806, + "rewards/cosine_scaled_reward": 0.10970486886799335, + "step": 131 + }, + { + "clip_fraction": 0.0, + "completion_length": 2745.8541870117188, + "epoch": 0.15085714285714286, + "grad_norm": 0.10217073559761047, + "kl": 3.91155481338501e-05, + "lambda_div_used": 0.5872602090239525, + "learning_rate": 3.8873442270461485e-07, + "loss": -0.0282, + "reward": -0.18162552546709776, + "reward_after_mean": -0.18162552546709776, + "reward_after_std": 0.4464551955461502, + "reward_before_mean": 0.14102918095886707, + "reward_before_std": 0.3960893382318318, + "reward_change_max": 0.0, + "reward_change_mean": -0.32265469804406166, + "reward_change_min": -0.477376826107502, + "reward_change_std": 0.18945662677288055, + "reward_std": 0.4464551992714405, + "rewards/accuracy_reward": 0.22916667722165585, + "rewards/cosine_scaled_reward": -0.08813749923137948, + "step": 132 + }, + { + "clip_fraction": 0.0, + "completion_length": 3396.625, + "epoch": 0.152, + "grad_norm": 0.05338770151138306, + "kl": 4.1738152503967285e-05, + "lambda_div_used": 0.5715877488255501, + "learning_rate": 3.8142703296283953e-07, + "loss": -0.0083, + "reward": -0.3019852191209793, + "reward_after_mean": -0.3019852191209793, + "reward_after_std": 0.394625848159194, + "reward_before_mean": 0.015070955269038677, + "reward_before_std": 0.3251611590385437, + "reward_change_max": 0.0, + "reward_change_mean": -0.3170561697334051, + "reward_change_min": -0.5240310952067375, + "reward_change_std": 0.18560713436454535, + "reward_std": 0.39462585374712944, + "rewards/accuracy_reward": 0.1666666679084301, + "rewards/cosine_scaled_reward": -0.15159572381526232, + "step": 133 + }, + { + "clip_fraction": 0.0, + "completion_length": 2655.7708435058594, + "epoch": 0.15314285714285714, + "grad_norm": 0.08679015934467316, + "kl": 8.308887481689453e-05, + "lambda_div_used": 0.5542216002941132, + "learning_rate": 3.7417099217982686e-07, + "loss": 0.0208, + "reward": -0.32178156822919846, + "reward_after_mean": -0.32178156822919846, + "reward_after_std": 0.2957470379769802, + "reward_before_mean": 0.012201536446809769, + "reward_before_std": 0.24192149471491575, + "reward_change_max": 0.0, + "reward_change_mean": -0.3339830953627825, + "reward_change_min": -0.4839101508259773, + "reward_change_std": 0.18742438219487667, + "reward_std": 0.2957470417022705, + "rewards/accuracy_reward": 0.1041666716337204, + "rewards/cosine_scaled_reward": -0.09196514077484608, + "step": 134 + }, + { + "clip_fraction": 0.0, + "completion_length": 2014.9583587646484, + "epoch": 0.15428571428571428, + "grad_norm": 0.07728467881679535, + "kl": 2.7412548661231995e-05, + "lambda_div_used": 0.6218282878398895, + "learning_rate": 3.6696851061588994e-07, + "loss": 0.0739, + "reward": 0.15841423906385899, + "reward_after_mean": 0.15841423906385899, + "reward_after_std": 0.6740392297506332, + "reward_before_mean": 0.6407645158469677, + "reward_before_std": 0.5697397403419018, + "reward_change_max": 0.0, + "reward_change_mean": -0.48235028609633446, + "reward_change_min": -0.7837426699697971, + "reward_change_std": 0.3034245353192091, + "reward_std": 0.6740392409265041, + "rewards/accuracy_reward": 0.4791666716337204, + "rewards/cosine_scaled_reward": 0.161597837228328, + "step": 135 + }, + { + "clip_fraction": 0.0, + "completion_length": 2602.416717529297, + "epoch": 0.15542857142857142, + "grad_norm": 0.06872163712978363, + "kl": 2.8759241104125977e-05, + "lambda_div_used": 0.6318690255284309, + "learning_rate": 3.5982178221668533e-07, + "loss": 0.0344, + "reward": 0.06710329907946289, + "reward_after_mean": 0.06710329907946289, + "reward_after_std": 0.6661366745829582, + "reward_before_mean": 0.454304663464427, + "reward_before_std": 0.6177845690399408, + "reward_change_max": 0.0, + "reward_change_mean": -0.38720136508345604, + "reward_change_min": -0.6763367056846619, + "reward_change_std": 0.258380358107388, + "reward_std": 0.6661366857588291, + "rewards/accuracy_reward": 0.3541666716337204, + "rewards/cosine_scaled_reward": 0.10013799648731947, + "step": 136 + }, + { + "clip_fraction": 0.0, + "completion_length": 3239.666679382324, + "epoch": 0.15657142857142858, + "grad_norm": 0.0593891404569149, + "kl": 3.842264413833618e-05, + "lambda_div_used": 0.5779730677604675, + "learning_rate": 3.5273298394491515e-07, + "loss": 0.0124, + "reward": -0.3537064976990223, + "reward_after_mean": -0.3537064976990223, + "reward_after_std": 0.41078851372003555, + "reward_before_mean": -0.08445308171212673, + "reward_before_std": 0.35645375214517117, + "reward_change_max": 0.0, + "reward_change_mean": -0.2692534364759922, + "reward_change_min": -0.4356107972562313, + "reward_change_std": 0.15999021660536528, + "reward_std": 0.4107885267585516, + "rewards/accuracy_reward": 0.08333333395421505, + "rewards/cosine_scaled_reward": -0.16778641194105148, + "step": 137 + }, + { + "clip_fraction": 0.0, + "completion_length": 2760.687526702881, + "epoch": 0.15771428571428572, + "grad_norm": 0.0790812149643898, + "kl": 2.8595328330993652e-05, + "lambda_div_used": 0.5799111127853394, + "learning_rate": 3.45704275117204e-07, + "loss": -0.0362, + "reward": -0.27829512720927596, + "reward_after_mean": -0.27829512720927596, + "reward_after_std": 0.41231589019298553, + "reward_before_mean": 0.02187724970281124, + "reward_before_std": 0.362152349203825, + "reward_change_max": 0.0, + "reward_change_mean": -0.3001723885536194, + "reward_change_min": -0.45827802270650864, + "reward_change_std": 0.17282870691269636, + "reward_std": 0.412315895780921, + "rewards/accuracy_reward": 0.12500000558793545, + "rewards/cosine_scaled_reward": -0.10312275495380163, + "step": 138 + }, + { + "clip_fraction": 0.0, + "completion_length": 2984.4583587646484, + "epoch": 0.15885714285714286, + "grad_norm": 0.06229131296277046, + "kl": 3.9517879486083984e-05, + "lambda_div_used": 0.5838895812630653, + "learning_rate": 3.387377967463493e-07, + "loss": -0.0589, + "reward": -0.3416835393290967, + "reward_after_mean": -0.3416835393290967, + "reward_after_std": 0.4420221708714962, + "reward_before_mean": -0.08248884417116642, + "reward_before_std": 0.384817186743021, + "reward_change_max": 0.0, + "reward_change_mean": -0.259194690734148, + "reward_change_min": -0.42579447478055954, + "reward_change_std": 0.15513746719807386, + "reward_std": 0.44202217273414135, + "rewards/accuracy_reward": 0.08333333395421505, + "rewards/cosine_scaled_reward": -0.16582217533141375, + "step": 139 + }, + { + "clip_fraction": 0.0, + "completion_length": 2881.1250228881836, + "epoch": 0.16, + "grad_norm": 0.071082703769207, + "kl": 4.310905933380127e-05, + "lambda_div_used": 0.5740059092640877, + "learning_rate": 3.3183567088914833e-07, + "loss": 0.0019, + "reward": -0.25879082828760147, + "reward_after_mean": -0.25879082828760147, + "reward_after_std": 0.4811950568109751, + "reward_before_mean": 0.09872006997466087, + "reward_before_std": 0.3372529884800315, + "reward_change_max": 0.0, + "reward_change_mean": -0.35751091688871384, + "reward_change_min": -0.4904663935303688, + "reward_change_std": 0.18400432635098696, + "reward_std": 0.481195081025362, + "rewards/accuracy_reward": 0.1666666679084301, + "rewards/cosine_scaled_reward": -0.06794659808656434, + "step": 140 + }, + { + "clip_fraction": 0.0, + "completion_length": 3180.187515258789, + "epoch": 0.16114285714285714, + "grad_norm": 0.05953420698642731, + "kl": 3.790855407714844e-05, + "lambda_div_used": 0.5660471692681313, + "learning_rate": 3.250000000000001e-07, + "loss": -0.0557, + "reward": -0.3696493972092867, + "reward_after_mean": -0.3696493972092867, + "reward_after_std": 0.37020369805395603, + "reward_before_mean": -0.1031316639855504, + "reward_before_std": 0.300534725189209, + "reward_change_max": 0.0, + "reward_change_mean": -0.2665177509188652, + "reward_change_min": -0.3948013670742512, + "reward_change_std": 0.151769301854074, + "reward_std": 0.3702037110924721, + "rewards/accuracy_reward": 0.10416666977107525, + "rewards/cosine_scaled_reward": -0.20729831932112575, + "step": 141 + }, + { + "clip_fraction": 0.0, + "completion_length": 2633.291732788086, + "epoch": 0.16228571428571428, + "grad_norm": 0.06993869692087173, + "kl": 4.523620009422302e-05, + "lambda_div_used": 0.643665611743927, + "learning_rate": 3.182328662904756e-07, + "loss": 0.0607, + "reward": -0.11501466785557568, + "reward_after_mean": -0.11501466785557568, + "reward_after_std": 0.7061873953789473, + "reward_before_mean": 0.1335596082135453, + "reward_before_std": 0.6679177191108465, + "reward_change_max": 0.0, + "reward_change_mean": -0.2485742662101984, + "reward_change_min": -0.438314501196146, + "reward_change_std": 0.16557594947516918, + "reward_std": 0.7061873972415924, + "rewards/accuracy_reward": 0.18750000186264515, + "rewards/cosine_scaled_reward": -0.05394040001556277, + "step": 142 + }, + { + "clip_fraction": 0.0, + "completion_length": 2543.854202270508, + "epoch": 0.16342857142857142, + "grad_norm": 0.09050207585096359, + "kl": 4.5418739318847656e-05, + "lambda_div_used": 0.5568482205271721, + "learning_rate": 3.115363310950578e-07, + "loss": 0.0559, + "reward": -0.44561293721199036, + "reward_after_mean": -0.44561293721199036, + "reward_after_std": 0.34061737172305584, + "reward_before_mean": -0.17733613029122353, + "reward_before_std": 0.2554386807605624, + "reward_change_max": 0.0, + "reward_change_mean": -0.26827680319547653, + "reward_change_min": -0.39999108389019966, + "reward_change_std": 0.14368562400341034, + "reward_std": 0.3406173773109913, + "rewards/accuracy_reward": 0.02083333395421505, + "rewards/cosine_scaled_reward": -0.19816946546779945, + "step": 143 + }, + { + "clip_fraction": 0.0, + "completion_length": 3062.2291984558105, + "epoch": 0.16457142857142856, + "grad_norm": 0.08773668855428696, + "kl": 4.3526291847229004e-05, + "lambda_div_used": 0.5946916490793228, + "learning_rate": 3.0491243424323783e-07, + "loss": 0.0214, + "reward": -0.1852840557694435, + "reward_after_mean": -0.1852840557694435, + "reward_after_std": 0.4624195992946625, + "reward_before_mean": 0.13713636994361877, + "reward_before_std": 0.4314339701086283, + "reward_change_max": 0.0, + "reward_change_mean": -0.3224204182624817, + "reward_change_min": -0.502110667526722, + "reward_change_std": 0.19803205784410238, + "reward_std": 0.4624196030199528, + "rewards/accuracy_reward": 0.18750000558793545, + "rewards/cosine_scaled_reward": -0.05036364169791341, + "step": 144 + }, + { + "clip_fraction": 0.0, + "completion_length": 2343.708381652832, + "epoch": 0.1657142857142857, + "grad_norm": 0.09618457406759262, + "kl": 3.914535045623779e-05, + "lambda_div_used": 0.6065893918275833, + "learning_rate": 2.9836319343816397e-07, + "loss": 0.0239, + "reward": -0.08643046766519547, + "reward_after_mean": -0.08643046766519547, + "reward_after_std": 0.565411014482379, + "reward_before_mean": 0.2700228439643979, + "reward_before_std": 0.4942969996482134, + "reward_change_max": 0.0, + "reward_change_mean": -0.35645330883562565, + "reward_change_min": -0.5621693357825279, + "reward_change_std": 0.21702316030859947, + "reward_std": 0.565411014482379, + "rewards/accuracy_reward": 0.29166666977107525, + "rewards/cosine_scaled_reward": -0.021643826737999916, + "step": 145 + }, + { + "clip_fraction": 0.0, + "completion_length": 2643.750030517578, + "epoch": 0.16685714285714287, + "grad_norm": 0.06749647855758667, + "kl": 3.0444934964179993e-05, + "lambda_div_used": 0.5759064182639122, + "learning_rate": 2.918906036420294e-07, + "loss": -0.0525, + "reward": -0.3986330684274435, + "reward_after_mean": -0.3986330684274435, + "reward_after_std": 0.4206914007663727, + "reward_before_mean": -0.1481835450977087, + "reward_before_std": 0.3465117560699582, + "reward_change_max": 0.0, + "reward_change_mean": -0.2504495307803154, + "reward_change_min": -0.3944549411535263, + "reward_change_std": 0.14348686579614878, + "reward_std": 0.42069140635430813, + "rewards/accuracy_reward": 0.06250000186264515, + "rewards/cosine_scaled_reward": -0.21068353950977325, + "step": 146 + }, + { + "clip_fraction": 0.0, + "completion_length": 3530.8958435058594, + "epoch": 0.168, + "grad_norm": 0.05082716792821884, + "kl": 4.493445158004761e-05, + "lambda_div_used": 0.6231074929237366, + "learning_rate": 2.854966364683872e-07, + "loss": 0.0064, + "reward": -0.17444764077663422, + "reward_after_mean": -0.17444764077663422, + "reward_after_std": 0.5966980569064617, + "reward_before_mean": 0.08711316343396902, + "reward_before_std": 0.5713155549019575, + "reward_change_max": 0.0, + "reward_change_mean": -0.26156080327928066, + "reward_change_min": -0.42396802455186844, + "reward_change_std": 0.16937424894422293, + "reward_std": 0.5966980736702681, + "rewards/accuracy_reward": 0.2083333395421505, + "rewards/cosine_scaled_reward": -0.12122016958892345, + "step": 147 + }, + { + "clip_fraction": 0.0, + "completion_length": 2726.0625228881836, + "epoch": 0.16914285714285715, + "grad_norm": 0.05811236426234245, + "kl": 2.9481947422027588e-05, + "lambda_div_used": 0.5576904863119125, + "learning_rate": 2.791832395815782e-07, + "loss": -0.0067, + "reward": -0.3092116080224514, + "reward_after_mean": -0.3092116080224514, + "reward_after_std": 0.3737166114151478, + "reward_before_mean": 0.04292176803573966, + "reward_before_std": 0.2566564744338393, + "reward_change_max": 0.0, + "reward_change_mean": -0.35213335789740086, + "reward_change_min": -0.5105132721364498, + "reward_change_std": 0.18572800233960152, + "reward_std": 0.3737166281789541, + "rewards/accuracy_reward": 0.14583333395421505, + "rewards/cosine_scaled_reward": -0.10291159152984619, + "step": 148 + }, + { + "clip_fraction": 0.0, + "completion_length": 3018.4375228881836, + "epoch": 0.1702857142857143, + "grad_norm": 0.07346916198730469, + "kl": 3.395974636077881e-05, + "lambda_div_used": 0.5993617027997971, + "learning_rate": 2.729523361034538e-07, + "loss": 0.0316, + "reward": -0.23622393235564232, + "reward_after_mean": -0.23622393235564232, + "reward_after_std": 0.513044873252511, + "reward_before_mean": 0.05197374615818262, + "reward_before_std": 0.4586914679966867, + "reward_change_max": 0.0, + "reward_change_mean": -0.2881976682692766, + "reward_change_min": -0.4784680940210819, + "reward_change_std": 0.1794019928202033, + "reward_std": 0.5130448862910271, + "rewards/accuracy_reward": 0.14583333767950535, + "rewards/cosine_scaled_reward": -0.0938595961779356, + "step": 149 + }, + { + "clip_fraction": 0.0, + "completion_length": 2817.000030517578, + "epoch": 0.17142857142857143, + "grad_norm": 0.07110631465911865, + "kl": 4.3623149394989014e-05, + "lambda_div_used": 0.5749505385756493, + "learning_rate": 2.6680582402757324e-07, + "loss": 0.0143, + "reward": -0.4273622464388609, + "reward_after_mean": -0.4273622464388609, + "reward_after_std": 0.41951365023851395, + "reward_before_mean": -0.1882035918533802, + "reward_before_std": 0.34118228033185005, + "reward_change_max": 0.0, + "reward_change_mean": -0.23915864899754524, + "reward_change_min": -0.3481520377099514, + "reward_change_std": 0.1273820260539651, + "reward_std": 0.4195136558264494, + "rewards/accuracy_reward": 0.0416666679084301, + "rewards/cosine_scaled_reward": -0.22987026162445545, + "step": 150 + }, + { + "clip_fraction": 0.0, + "completion_length": 2473.5208892822266, + "epoch": 0.17257142857142857, + "grad_norm": 0.06608390063047409, + "kl": 3.757700324058533e-05, + "lambda_div_used": 0.6444682851433754, + "learning_rate": 2.6074557564105724e-07, + "loss": 0.0674, + "reward": 0.013197226449847221, + "reward_after_mean": 0.013197226449847221, + "reward_after_std": 0.6638381816446781, + "reward_before_mean": 0.3273471943102777, + "reward_before_std": 0.6709178425371647, + "reward_change_max": 0.0, + "reward_change_mean": -0.31414996832609177, + "reward_change_min": -0.5566080771386623, + "reward_change_std": 0.22394540812820196, + "reward_std": 0.6638382077217102, + "rewards/accuracy_reward": 0.31250000558793545, + "rewards/cosine_scaled_reward": 0.014847185462713242, + "step": 151 + }, + { + "clip_fraction": 0.0, + "completion_length": 2980.312530517578, + "epoch": 0.1737142857142857, + "grad_norm": 0.09729959070682526, + "kl": 3.904849290847778e-05, + "lambda_div_used": 0.5731147676706314, + "learning_rate": 2.547734369542718e-07, + "loss": 0.036, + "reward": -0.42251406982541084, + "reward_after_mean": -0.42251406982541084, + "reward_after_std": 0.4107065536081791, + "reward_before_mean": -0.17358114942908287, + "reward_before_std": 0.3313267915509641, + "reward_change_max": 0.0, + "reward_change_mean": -0.24893292412161827, + "reward_change_min": -0.37377795577049255, + "reward_change_std": 0.13549237046390772, + "reward_std": 0.4107065834105015, + "rewards/accuracy_reward": 0.0416666679084301, + "rewards/cosine_scaled_reward": -0.21524782478809357, + "step": 152 + }, + { + "clip_fraction": 0.0, + "completion_length": 2980.4167098999023, + "epoch": 0.17485714285714285, + "grad_norm": 0.07872413098812103, + "kl": 5.060434341430664e-05, + "lambda_div_used": 0.5996033921837807, + "learning_rate": 2.488912271385139e-07, + "loss": 0.0468, + "reward": -0.25326492823660374, + "reward_after_mean": -0.25326492823660374, + "reward_after_std": 0.5015600807964802, + "reward_before_mean": 0.03005093801766634, + "reward_before_std": 0.45544715132564306, + "reward_change_max": 0.0, + "reward_change_mean": -0.2833158541470766, + "reward_change_min": -0.490575835108757, + "reward_change_std": 0.17807927820831537, + "reward_std": 0.5015600919723511, + "rewards/accuracy_reward": 0.12500000186264515, + "rewards/cosine_scaled_reward": -0.09494908014312387, + "step": 153 + }, + { + "clip_fraction": 0.0, + "completion_length": 3400.6250610351562, + "epoch": 0.176, + "grad_norm": 0.044398024678230286, + "kl": 3.533065319061279e-05, + "lambda_div_used": 0.6407897770404816, + "learning_rate": 2.4310073797187573e-07, + "loss": 0.001, + "reward": -0.08846104983240366, + "reward_after_mean": -0.08846104983240366, + "reward_after_std": 0.6875216029584408, + "reward_before_mean": 0.1883750823326409, + "reward_before_std": 0.6521897967904806, + "reward_change_max": 0.0, + "reward_change_mean": -0.27683611772954464, + "reward_change_min": -0.45937369018793106, + "reward_change_std": 0.17437504325062037, + "reward_std": 0.6875216346234083, + "rewards/accuracy_reward": 0.20833334140479565, + "rewards/cosine_scaled_reward": -0.019958254415541887, + "step": 154 + }, + { + "clip_fraction": 0.0, + "completion_length": 2453.5416717529297, + "epoch": 0.17714285714285713, + "grad_norm": 0.08257688581943512, + "kl": 4.6115368604660034e-05, + "lambda_div_used": 0.5891367048025131, + "learning_rate": 2.374037332934512e-07, + "loss": 0.0267, + "reward": -0.14017992746084929, + "reward_after_mean": -0.14017992746084929, + "reward_after_std": 0.5328723080456257, + "reward_before_mean": 0.24201755598187447, + "reward_before_std": 0.40873255487531424, + "reward_change_max": 0.0, + "reward_change_mean": -0.3821975253522396, + "reward_change_min": -0.5966118611395359, + "reward_change_std": 0.22055233176797628, + "reward_std": 0.532872324809432, + "rewards/accuracy_reward": 0.22916666977107525, + "rewards/cosine_scaled_reward": 0.012850900180637836, + "step": 155 + }, + { + "clip_fraction": 0.0, + "completion_length": 3058.145835876465, + "epoch": 0.1782857142857143, + "grad_norm": 0.07657067477703094, + "kl": 3.8407742977142334e-05, + "lambda_div_used": 0.5792115926742554, + "learning_rate": 2.3180194846605364e-07, + "loss": 0.0192, + "reward": -0.35964071936905384, + "reward_after_mean": -0.35964071936905384, + "reward_after_std": 0.41963592916727066, + "reward_before_mean": -0.10375695489346981, + "reward_before_std": 0.36636115331202745, + "reward_change_max": 0.0, + "reward_change_mean": -0.2558837812393904, + "reward_change_min": -0.40037716925144196, + "reward_change_std": 0.15252912789583206, + "reward_std": 0.41963593289256096, + "rewards/accuracy_reward": 0.08333333395421505, + "rewards/cosine_scaled_reward": -0.18709028977900743, + "step": 156 + }, + { + "clip_fraction": 0.0, + "completion_length": 3067.0833435058594, + "epoch": 0.17942857142857144, + "grad_norm": 0.0602630190551281, + "kl": 4.166364669799805e-05, + "lambda_div_used": 0.578017845749855, + "learning_rate": 2.2629708984760706e-07, + "loss": -0.0024, + "reward": -0.20843622065149248, + "reward_after_mean": -0.20843622065149248, + "reward_after_std": 0.46523306891322136, + "reward_before_mean": 0.15985593758523464, + "reward_before_std": 0.35562971234321594, + "reward_change_max": 0.0, + "reward_change_mean": -0.3682921752333641, + "reward_change_min": -0.5581395737826824, + "reward_change_std": 0.20453235507011414, + "reward_std": 0.46523308381438255, + "rewards/accuracy_reward": 0.18750000186264515, + "rewards/cosine_scaled_reward": -0.027644065208733082, + "step": 157 + }, + { + "clip_fraction": 0.0, + "completion_length": 2439.8125381469727, + "epoch": 0.18057142857142858, + "grad_norm": 0.07962260395288467, + "kl": 3.724917769432068e-05, + "lambda_div_used": 0.5700756460428238, + "learning_rate": 2.2089083427137329e-07, + "loss": 0.0427, + "reward": -0.09846613928675652, + "reward_after_mean": -0.09846613928675652, + "reward_after_std": 0.50458899512887, + "reward_before_mean": 0.36336963158100843, + "reward_before_std": 0.31559942476451397, + "reward_change_max": 0.0, + "reward_change_mean": -0.46183576062321663, + "reward_change_min": -0.6303520426154137, + "reward_change_std": 0.23663399182260036, + "reward_std": 0.5045890025794506, + "rewards/accuracy_reward": 0.2916666679084301, + "rewards/cosine_scaled_reward": 0.07170296460390091, + "step": 158 + }, + { + "clip_fraction": 0.0, + "completion_length": 3283.7916717529297, + "epoch": 0.18171428571428572, + "grad_norm": 0.05873558670282364, + "kl": 4.16487455368042e-05, + "lambda_div_used": 0.565159484744072, + "learning_rate": 2.1558482853517253e-07, + "loss": -0.0183, + "reward": -0.43611275404691696, + "reward_after_mean": -0.43611275404691696, + "reward_after_std": 0.34730882942676544, + "reward_before_mean": -0.18830033391714096, + "reward_before_std": 0.2990688029676676, + "reward_change_max": 0.0, + "reward_change_mean": -0.247812420129776, + "reward_change_min": -0.4116964153945446, + "reward_change_std": 0.15130725782364607, + "reward_std": 0.34730884805321693, + "rewards/accuracy_reward": 0.0625, + "rewards/cosine_scaled_reward": -0.2508003171533346, + "step": 159 + }, + { + "clip_fraction": 0.0, + "completion_length": 3132.4167098999023, + "epoch": 0.18285714285714286, + "grad_norm": 0.06507622450590134, + "kl": 5.078315734863281e-05, + "lambda_div_used": 0.5871393531560898, + "learning_rate": 2.1038068889975259e-07, + "loss": 0.0371, + "reward": -0.25536114536225796, + "reward_after_mean": -0.25536114536225796, + "reward_after_std": 0.4553165938705206, + "reward_before_mean": 0.0424603084102273, + "reward_before_std": 0.3940334524959326, + "reward_change_max": 0.0, + "reward_change_mean": -0.2978214379400015, + "reward_change_min": -0.43723243474960327, + "reward_change_std": 0.16810003202408552, + "reward_std": 0.45531659945845604, + "rewards/accuracy_reward": 0.1458333395421505, + "rewards/cosine_scaled_reward": -0.10337302926927805, + "step": 160 + }, + { + "clip_fraction": 0.0, + "completion_length": 1814.145866394043, + "epoch": 0.184, + "grad_norm": 0.09199656546115875, + "kl": 3.68654727935791e-05, + "lambda_div_used": 0.6221684664487839, + "learning_rate": 2.0528000059645995e-07, + "loss": -0.1313, + "reward": -0.14752347487956285, + "reward_after_mean": -0.14752347487956285, + "reward_after_std": 0.6091006584465504, + "reward_before_mean": 0.1306287944316864, + "reward_before_std": 0.5693164113909006, + "reward_change_max": 0.0, + "reward_change_mean": -0.2781522646546364, + "reward_change_min": -0.4651326909661293, + "reward_change_std": 0.17978444695472717, + "reward_std": 0.6091006807982922, + "rewards/accuracy_reward": 0.1666666679084301, + "rewards/cosine_scaled_reward": -0.03603787627071142, + "step": 161 + }, + { + "clip_fraction": 0.0, + "completion_length": 3445.2291870117188, + "epoch": 0.18514285714285714, + "grad_norm": 0.054613932967185974, + "kl": 4.4696033000946045e-05, + "lambda_div_used": 0.6093570217490196, + "learning_rate": 2.0028431734436308e-07, + "loss": 0.0111, + "reward": -0.21708323806524277, + "reward_after_mean": -0.21708323806524277, + "reward_after_std": 0.5340255293995142, + "reward_before_mean": 0.05781930312514305, + "reward_before_std": 0.5134020620025694, + "reward_change_max": 0.0, + "reward_change_mean": -0.27490255795419216, + "reward_change_min": -0.5003471188247204, + "reward_change_std": 0.1919182576239109, + "reward_std": 0.5340255443006754, + "rewards/accuracy_reward": 0.1666666679084301, + "rewards/cosine_scaled_reward": -0.10884736478328705, + "step": 162 + }, + { + "clip_fraction": 0.0, + "completion_length": 2758.5833435058594, + "epoch": 0.18628571428571428, + "grad_norm": 0.06869502365589142, + "kl": 3.579258918762207e-05, + "lambda_div_used": 0.5997234806418419, + "learning_rate": 1.9539516087697517e-07, + "loss": 0.025, + "reward": -0.0028839372098445892, + "reward_after_mean": -0.0028839372098445892, + "reward_after_std": 0.5558424014598131, + "reward_before_mean": 0.42010904336348176, + "reward_before_std": 0.46184782730415463, + "reward_change_max": 0.0, + "reward_change_mean": -0.42299298755824566, + "reward_change_min": -0.614623662084341, + "reward_change_std": 0.2444069180637598, + "reward_std": 0.5558424014598131, + "rewards/accuracy_reward": 0.33333334140479565, + "rewards/cosine_scaled_reward": 0.08677570521831512, + "step": 163 + }, + { + "clip_fraction": 0.0, + "completion_length": 2280.4583435058594, + "epoch": 0.18742857142857142, + "grad_norm": 0.08430740982294083, + "kl": 3.807246685028076e-05, + "lambda_div_used": 0.583802655339241, + "learning_rate": 1.9061402047871833e-07, + "loss": -0.0005, + "reward": -0.14323966577649117, + "reward_after_mean": -0.14323966577649117, + "reward_after_std": 0.48269340209662914, + "reward_before_mean": 0.24511565826833248, + "reward_before_std": 0.38433590345084667, + "reward_change_max": 0.0, + "reward_change_mean": -0.38835531286895275, + "reward_change_min": -0.6137315705418587, + "reward_change_std": 0.2298044739291072, + "reward_std": 0.48269341699779034, + "rewards/accuracy_reward": 0.2500000037252903, + "rewards/cosine_scaled_reward": -0.0048843612894415855, + "step": 164 + }, + { + "clip_fraction": 0.0, + "completion_length": 3350.2083435058594, + "epoch": 0.18857142857142858, + "grad_norm": 0.06055794283747673, + "kl": 4.6700239181518555e-05, + "lambda_div_used": 0.5597026646137238, + "learning_rate": 1.8594235253127372e-07, + "loss": -0.0239, + "reward": -0.383526224642992, + "reward_after_mean": -0.383526224642992, + "reward_after_std": 0.3388876337558031, + "reward_before_mean": -0.09019916784018278, + "reward_before_std": 0.27148230001330376, + "reward_change_max": 0.0, + "reward_change_mean": -0.2933270577341318, + "reward_change_min": -0.44717802107334137, + "reward_change_std": 0.16905023716390133, + "reward_std": 0.3388876374810934, + "rewards/accuracy_reward": 0.0625, + "rewards/cosine_scaled_reward": -0.15269917901605368, + "step": 165 + }, + { + "clip_fraction": 0.0, + "completion_length": 2993.7709045410156, + "epoch": 0.18971428571428572, + "grad_norm": 0.07070616632699966, + "kl": 3.575533628463745e-05, + "lambda_div_used": 0.6159055009484291, + "learning_rate": 1.8138158006995363e-07, + "loss": -0.1171, + "reward": -0.20591574627906084, + "reward_after_mean": -0.20591574627906084, + "reward_after_std": 0.568760309368372, + "reward_before_mean": 0.07225125166587532, + "reward_before_std": 0.5325462874025106, + "reward_change_max": 0.0, + "reward_change_mean": -0.2781669981777668, + "reward_change_min": -0.4841184914112091, + "reward_change_std": 0.17965693771839142, + "reward_std": 0.5687603335827589, + "rewards/accuracy_reward": 0.1458333358168602, + "rewards/cosine_scaled_reward": -0.0735820853151381, + "step": 166 + }, + { + "clip_fraction": 0.0, + "completion_length": 2243.4583740234375, + "epoch": 0.19085714285714286, + "grad_norm": 0.08954203128814697, + "kl": 3.8199592381715775e-05, + "lambda_div_used": 0.5807118713855743, + "learning_rate": 1.7693309235023127e-07, + "loss": -0.0227, + "reward": -0.12234261445701122, + "reward_after_mean": -0.12234261445701122, + "reward_after_std": 0.46423870138823986, + "reward_before_mean": 0.28887630719691515, + "reward_before_std": 0.3718760753981769, + "reward_change_max": 0.0, + "reward_change_mean": -0.41121890768408775, + "reward_change_min": -0.6246605962514877, + "reward_change_std": 0.24211041443049908, + "reward_std": 0.4642387069761753, + "rewards/accuracy_reward": 0.2291666679084301, + "rewards/cosine_scaled_reward": 0.0597096448764205, + "step": 167 + }, + { + "clip_fraction": 0.0, + "completion_length": 3159.604217529297, + "epoch": 0.192, + "grad_norm": 0.05736541375517845, + "kl": 4.6193599700927734e-05, + "lambda_div_used": 0.6025644987821579, + "learning_rate": 1.7259824442455923e-07, + "loss": 0.0616, + "reward": -0.1081604603677988, + "reward_after_mean": -0.1081604603677988, + "reward_after_std": 0.5043755322694778, + "reward_before_mean": 0.23938040900975466, + "reward_before_std": 0.4762058244086802, + "reward_change_max": 0.0, + "reward_change_mean": -0.347540894523263, + "reward_change_min": -0.5797867476940155, + "reward_change_std": 0.22277529910206795, + "reward_std": 0.504375534132123, + "rewards/accuracy_reward": 0.2083333395421505, + "rewards/cosine_scaled_reward": 0.031047106254845858, + "step": 168 + }, + { + "clip_fraction": 0.0, + "completion_length": 2691.166717529297, + "epoch": 0.19314285714285714, + "grad_norm": 0.05840221792459488, + "kl": 3.322819247841835e-05, + "lambda_div_used": 0.6123679727315903, + "learning_rate": 1.6837835672960831e-07, + "loss": -0.0221, + "reward": 0.15223310515284538, + "reward_after_mean": 0.15223310515284538, + "reward_after_std": 0.589836286380887, + "reward_before_mean": 0.622445510700345, + "reward_before_std": 0.5133083704859018, + "reward_change_max": 0.0, + "reward_change_mean": -0.4702123887836933, + "reward_change_min": -0.686709500849247, + "reward_change_std": 0.27231256756931543, + "reward_std": 0.5898362882435322, + "rewards/accuracy_reward": 0.45833334140479565, + "rewards/cosine_scaled_reward": 0.1641121432185173, + "step": 169 + }, + { + "clip_fraction": 0.0, + "completion_length": 2737.8542137145996, + "epoch": 0.19428571428571428, + "grad_norm": 0.08408840745687485, + "kl": 2.9318034648895264e-05, + "lambda_div_used": 0.5516072362661362, + "learning_rate": 1.6427471468404952e-07, + "loss": -0.0223, + "reward": -0.22534361481666565, + "reward_after_mean": -0.22534361481666565, + "reward_after_std": 0.34615582413971424, + "reward_before_mean": 0.17829649709165096, + "reward_before_std": 0.231270051561296, + "reward_change_max": 0.0, + "reward_change_mean": -0.4036401268094778, + "reward_change_min": -0.5518503561615944, + "reward_change_std": 0.21789801493287086, + "reward_std": 0.34615583159029484, + "rewards/accuracy_reward": 0.2291666716337204, + "rewards/cosine_scaled_reward": -0.05087016709148884, + "step": 170 + }, + { + "clip_fraction": 0.0, + "completion_length": 2810.208366394043, + "epoch": 0.19542857142857142, + "grad_norm": 0.07381287217140198, + "kl": 3.468245267868042e-05, + "lambda_div_used": 0.6072571501135826, + "learning_rate": 1.6028856829700258e-07, + "loss": 0.0763, + "reward": -0.0880913995206356, + "reward_after_mean": -0.0880913995206356, + "reward_after_std": 0.4982736185193062, + "reward_before_mean": 0.2392272837460041, + "reward_before_std": 0.4966685324907303, + "reward_change_max": 0.0, + "reward_change_mean": -0.3273186720907688, + "reward_change_min": -0.5419761650264263, + "reward_change_std": 0.21660141553729773, + "reward_std": 0.4982736259698868, + "rewards/accuracy_reward": 0.2291666716337204, + "rewards/cosine_scaled_reward": 0.010060586035251617, + "step": 171 + }, + { + "clip_fraction": 0.0, + "completion_length": 2782.833381652832, + "epoch": 0.19657142857142856, + "grad_norm": 0.08758535981178284, + "kl": 4.4949352741241455e-05, + "lambda_div_used": 0.5883520841598511, + "learning_rate": 1.5642113178727193e-07, + "loss": 0.0173, + "reward": -0.0421207002364099, + "reward_after_mean": -0.0421207002364099, + "reward_after_std": 0.5838102325797081, + "reward_before_mean": 0.41242535319179296, + "reward_before_std": 0.4052962730638683, + "reward_change_max": 0.0, + "reward_change_mean": -0.45454608276486397, + "reward_change_min": -0.6413811855018139, + "reward_change_std": 0.2437124690040946, + "reward_std": 0.5838102586567402, + "rewards/accuracy_reward": 0.3333333358168602, + "rewards/cosine_scaled_reward": 0.07909201784059405, + "step": 172 + }, + { + "clip_fraction": 0.0, + "completion_length": 2038.3125114440918, + "epoch": 0.1977142857142857, + "grad_norm": 0.10115820169448853, + "kl": 2.4806708097457886e-05, + "lambda_div_used": 0.5800783261656761, + "learning_rate": 1.5267358321348285e-07, + "loss": -0.0379, + "reward": -0.3465901352465153, + "reward_after_mean": -0.3465901352465153, + "reward_after_std": 0.41013904474675655, + "reward_before_mean": -0.07510924944654107, + "reward_before_std": 0.3657920900732279, + "reward_change_max": 0.0, + "reward_change_mean": -0.27148088812828064, + "reward_change_min": -0.41459682397544384, + "reward_change_std": 0.16112061124294996, + "reward_std": 0.4101390540599823, + "rewards/accuracy_reward": 0.10416666977107525, + "rewards/cosine_scaled_reward": -0.17927592061460018, + "step": 173 + }, + { + "clip_fraction": 0.0, + "completion_length": 2429.7083740234375, + "epoch": 0.19885714285714284, + "grad_norm": 0.08579805493354797, + "kl": 4.820525646209717e-05, + "lambda_div_used": 0.5971189364790916, + "learning_rate": 1.4904706411523448e-07, + "loss": -0.0681, + "reward": -0.16348301246762276, + "reward_after_mean": -0.16348301246762276, + "reward_after_std": 0.5588766317814589, + "reward_before_mean": 0.18685074103996158, + "reward_before_std": 0.44327013567090034, + "reward_change_max": 0.0, + "reward_change_mean": -0.3503337763249874, + "reward_change_min": -0.5374101847410202, + "reward_change_std": 0.19548403285443783, + "reward_std": 0.5588766410946846, + "rewards/accuracy_reward": 0.2083333358168602, + "rewards/cosine_scaled_reward": -0.02148258499801159, + "step": 174 + }, + { + "clip_fraction": 0.0, + "completion_length": 2815.1458587646484, + "epoch": 0.2, + "grad_norm": 0.06386198848485947, + "kl": 3.6529265344142914e-05, + "lambda_div_used": 0.5798816308379173, + "learning_rate": 1.4554267916537495e-07, + "loss": 0.0362, + "reward": -0.08284463733434677, + "reward_after_mean": -0.08284463733434677, + "reward_after_std": 0.46138195879757404, + "reward_before_mean": 0.33735317550599575, + "reward_before_std": 0.3691369164735079, + "reward_change_max": 0.0, + "reward_change_mean": -0.4201977960765362, + "reward_change_min": -0.615710511803627, + "reward_change_std": 0.2454231232404709, + "reward_std": 0.46138197369873524, + "rewards/accuracy_reward": 0.2708333358168602, + "rewards/cosine_scaled_reward": 0.06651980988681316, + "step": 175 + }, + { + "clip_fraction": 0.0, + "completion_length": 2699.916748046875, + "epoch": 0.20114285714285715, + "grad_norm": 0.1062018871307373, + "kl": 3.183633089065552e-05, + "lambda_div_used": 0.6652076914906502, + "learning_rate": 1.4216149583350755e-07, + "loss": -0.0096, + "reward": 0.01935443957336247, + "reward_after_mean": 0.01935443957336247, + "reward_after_std": 0.7740565538406372, + "reward_before_mean": 0.30571601539850235, + "reward_before_std": 0.7710783276706934, + "reward_change_max": 0.0, + "reward_change_mean": -0.2863615807145834, + "reward_change_min": -0.5516976863145828, + "reward_change_std": 0.20931501779705286, + "reward_std": 0.7740565687417984, + "rewards/accuracy_reward": 0.2708333395421505, + "rewards/cosine_scaled_reward": 0.03488267329521477, + "step": 176 + }, + { + "clip_fraction": 0.0, + "completion_length": 2837.062530517578, + "epoch": 0.2022857142857143, + "grad_norm": 0.10419050604104996, + "kl": 5.3942203521728516e-05, + "lambda_div_used": 0.5891516581177711, + "learning_rate": 1.3890454406082956e-07, + "loss": -0.094, + "reward": -0.275299109518528, + "reward_after_mean": -0.275299109518528, + "reward_after_std": 0.4370743464678526, + "reward_before_mean": 0.019260672852396965, + "reward_before_std": 0.4135833829641342, + "reward_change_max": 0.0, + "reward_change_mean": -0.294559795409441, + "reward_change_min": -0.49321193993091583, + "reward_change_std": 0.19294326566159725, + "reward_std": 0.43707435205578804, + "rewards/accuracy_reward": 0.125, + "rewards/cosine_scaled_reward": -0.10573932714760303, + "step": 177 + }, + { + "clip_fraction": 0.0, + "completion_length": 2239.645866394043, + "epoch": 0.20342857142857143, + "grad_norm": 0.08482369780540466, + "kl": 3.471970558166504e-05, + "lambda_div_used": 0.5743999034166336, + "learning_rate": 1.3577281594640182e-07, + "loss": -0.012, + "reward": -0.2907655192539096, + "reward_after_mean": -0.2907655192539096, + "reward_after_std": 0.3906351812183857, + "reward_before_mean": 0.020154590718448162, + "reward_before_std": 0.33835357427597046, + "reward_change_max": 0.0, + "reward_change_mean": -0.31092010997235775, + "reward_change_min": -0.4689374789595604, + "reward_change_std": 0.18138453178107738, + "reward_std": 0.3906352035701275, + "rewards/accuracy_reward": 0.10416666977107525, + "rewards/cosine_scaled_reward": -0.08401207532733679, + "step": 178 + }, + { + "clip_fraction": 0.0, + "completion_length": 2847.187530517578, + "epoch": 0.20457142857142857, + "grad_norm": 0.06261342018842697, + "kl": 3.7359073758125305e-05, + "lambda_div_used": 0.5993921086192131, + "learning_rate": 1.3276726544494571e-07, + "loss": 0.0003, + "reward": -0.3251352347433567, + "reward_after_mean": -0.3251352347433567, + "reward_after_std": 0.507962841540575, + "reward_before_mean": -0.08432525303214788, + "reward_before_std": 0.45724861416965723, + "reward_change_max": 0.0, + "reward_change_mean": -0.2408099938184023, + "reward_change_min": -0.39610618352890015, + "reward_change_std": 0.14655023906379938, + "reward_std": 0.5079628489911556, + "rewards/accuracy_reward": 0.1041666679084301, + "rewards/cosine_scaled_reward": -0.18849190324544907, + "step": 179 + }, + { + "clip_fraction": 0.0, + "completion_length": 2329.562515258789, + "epoch": 0.2057142857142857, + "grad_norm": 0.10947566479444504, + "kl": 3.916025161743164e-05, + "lambda_div_used": 0.6209289953112602, + "learning_rate": 1.2988880807625927e-07, + "loss": -0.0732, + "reward": 0.0392537759616971, + "reward_after_mean": 0.0392537759616971, + "reward_after_std": 0.6374544408172369, + "reward_before_mean": 0.4333432329003699, + "reward_before_std": 0.5579905398190022, + "reward_change_max": 0.0, + "reward_change_mean": -0.3940894529223442, + "reward_change_min": -0.5780177563428879, + "reward_change_std": 0.229821746237576, + "reward_std": 0.6374544575810432, + "rewards/accuracy_reward": 0.3541666753590107, + "rewards/cosine_scaled_reward": 0.07917657308280468, + "step": 180 + }, + { + "clip_fraction": 0.0, + "completion_length": 3174.9166870117188, + "epoch": 0.20685714285714285, + "grad_norm": 0.0638444796204567, + "kl": 4.488229751586914e-05, + "lambda_div_used": 0.5625592544674873, + "learning_rate": 1.2713832064634125e-07, + "loss": -0.0055, + "reward": -0.3669877387583256, + "reward_after_mean": -0.3669877387583256, + "reward_after_std": 0.3254594895988703, + "reward_before_mean": -0.07308395206928253, + "reward_before_std": 0.2811311110854149, + "reward_change_max": 0.0, + "reward_change_mean": -0.29390377551317215, + "reward_change_min": -0.47421957924962044, + "reward_change_std": 0.1733461432158947, + "reward_std": 0.3254595026373863, + "rewards/accuracy_reward": 0.0833333358168602, + "rewards/cosine_scaled_reward": -0.15641729161143303, + "step": 181 + }, + { + "clip_fraction": 0.0, + "completion_length": 2200.5000228881836, + "epoch": 0.208, + "grad_norm": 0.07948590070009232, + "kl": 3.180652856826782e-05, + "lambda_div_used": 0.5802313759922981, + "learning_rate": 1.2451664098030743e-07, + "loss": -0.0177, + "reward": -0.33155214530415833, + "reward_after_mean": -0.33155214530415833, + "reward_after_std": 0.4093964695930481, + "reward_before_mean": -0.04757622070610523, + "reward_before_std": 0.362399042584002, + "reward_change_max": 0.0, + "reward_change_mean": -0.2839759271591902, + "reward_change_min": -0.4600135274231434, + "reward_change_std": 0.1762974951416254, + "reward_std": 0.40939648635685444, + "rewards/accuracy_reward": 0.1458333358168602, + "rewards/cosine_scaled_reward": -0.19340955652296543, + "step": 182 + }, + { + "clip_fraction": 0.0, + "completion_length": 1919.0000381469727, + "epoch": 0.20914285714285713, + "grad_norm": 0.09855654090642929, + "kl": 3.127008676528931e-05, + "lambda_div_used": 0.5876503959298134, + "learning_rate": 1.220245676671809e-07, + "loss": 0.0354, + "reward": -0.29830155335366726, + "reward_after_mean": -0.29830155335366726, + "reward_after_std": 0.461369052529335, + "reward_before_mean": -0.01361087104305625, + "reward_before_std": 0.4003021940588951, + "reward_change_max": 0.0, + "reward_change_mean": -0.28469069860875607, + "reward_change_min": -0.4400797598063946, + "reward_change_std": 0.1702390005812049, + "reward_std": 0.4613690562546253, + "rewards/accuracy_reward": 0.1041666679084301, + "rewards/cosine_scaled_reward": -0.11777753569185734, + "step": 183 + }, + { + "clip_fraction": 0.0, + "completion_length": 2771.0416870117188, + "epoch": 0.2102857142857143, + "grad_norm": 0.09264298528432846, + "kl": 4.6037137508392334e-05, + "lambda_div_used": 0.5391388088464737, + "learning_rate": 1.1966285981663407e-07, + "loss": -0.001, + "reward": -0.3703397810459137, + "reward_after_mean": -0.3703397810459137, + "reward_after_std": 0.2937673106789589, + "reward_before_mean": -0.011352727189660072, + "reward_before_std": 0.17188004031777382, + "reward_change_max": 0.0, + "reward_change_mean": -0.35898703522980213, + "reward_change_min": -0.5206632278859615, + "reward_change_std": 0.18913730140775442, + "reward_std": 0.29376731254160404, + "rewards/accuracy_reward": 0.125, + "rewards/cosine_scaled_reward": -0.13635273277759552, + "step": 184 + }, + { + "clip_fraction": 0.0, + "completion_length": 2770.9792098999023, + "epoch": 0.21142857142857144, + "grad_norm": 0.09669750928878784, + "kl": 4.3720006942749023e-05, + "lambda_div_used": 0.5636111497879028, + "learning_rate": 1.1743223682775649e-07, + "loss": 0.0651, + "reward": -0.3112166179344058, + "reward_after_mean": -0.3112166179344058, + "reward_after_std": 0.3695005215704441, + "reward_before_mean": -0.0006957156583666801, + "reward_before_std": 0.2874382403679192, + "reward_change_max": 0.0, + "reward_change_mean": -0.3105209097266197, + "reward_change_min": -0.43808290362358093, + "reward_change_std": 0.17091987561434507, + "reward_std": 0.369500532746315, + "rewards/accuracy_reward": 0.10416666977107525, + "rewards/cosine_scaled_reward": -0.10486237239092588, + "step": 185 + }, + { + "clip_fraction": 0.0, + "completion_length": 2965.375015258789, + "epoch": 0.21257142857142858, + "grad_norm": 0.05594424530863762, + "kl": 3.8955360651016235e-05, + "lambda_div_used": 0.608224630355835, + "learning_rate": 1.1533337816991931e-07, + "loss": -0.0681, + "reward": -0.20297073479741812, + "reward_after_mean": -0.20297073479741812, + "reward_after_std": 0.5205403883010149, + "reward_before_mean": 0.07951527182012796, + "reward_before_std": 0.502502404153347, + "reward_change_max": 0.0, + "reward_change_mean": -0.282486030831933, + "reward_change_min": -0.47177543863654137, + "reward_change_std": 0.18455488048493862, + "reward_std": 0.5205404032021761, + "rewards/accuracy_reward": 0.16666666977107525, + "rewards/cosine_scaled_reward": -0.08715140074491501, + "step": 186 + }, + { + "clip_fraction": 0.0, + "completion_length": 2639.291702270508, + "epoch": 0.21371428571428572, + "grad_norm": 0.08843009173870087, + "kl": 4.921853542327881e-05, + "lambda_div_used": 0.5908937901258469, + "learning_rate": 1.1336692317580158e-07, + "loss": -0.0048, + "reward": -0.25490788742899895, + "reward_after_mean": -0.25490788742899895, + "reward_after_std": 0.4807432759553194, + "reward_before_mean": 0.04253344633616507, + "reward_before_std": 0.4119319263845682, + "reward_change_max": 0.0, + "reward_change_mean": -0.29744134843349457, + "reward_change_min": -0.4517475329339504, + "reward_change_std": 0.16703799460083246, + "reward_std": 0.48074328526854515, + "rewards/accuracy_reward": 0.1458333395421505, + "rewards/cosine_scaled_reward": -0.10329987667500973, + "step": 187 + }, + { + "clip_fraction": 0.0, + "completion_length": 3541.9791870117188, + "epoch": 0.21485714285714286, + "grad_norm": 0.05120408907532692, + "kl": 4.652142524719238e-05, + "lambda_div_used": 0.5589732080698013, + "learning_rate": 1.1153347084664419e-07, + "loss": -0.0029, + "reward": -0.4253443591296673, + "reward_after_mean": -0.4253443591296673, + "reward_after_std": 0.3237063102424145, + "reward_before_mean": -0.15165027976036072, + "reward_before_std": 0.26862882915884256, + "reward_change_max": 0.0, + "reward_change_mean": -0.27369407564401627, + "reward_change_min": -0.437398936599493, + "reward_change_std": 0.16332428343594074, + "reward_std": 0.3237063158303499, + "rewards/accuracy_reward": 0.0625, + "rewards/cosine_scaled_reward": -0.2141502844169736, + "step": 188 + }, + { + "clip_fraction": 0.0, + "completion_length": 2377.2708892822266, + "epoch": 0.216, + "grad_norm": 0.07240846008062363, + "kl": 5.1606446504592896e-05, + "lambda_div_used": 0.5700332000851631, + "learning_rate": 1.0983357966978745e-07, + "loss": 0.0342, + "reward": -0.3627132559195161, + "reward_after_mean": -0.3627132559195161, + "reward_after_std": 0.4038649797439575, + "reward_before_mean": -0.08412502333521843, + "reward_before_std": 0.3190039964392781, + "reward_change_max": 0.0, + "reward_change_mean": -0.2785882242023945, + "reward_change_min": -0.4183661602437496, + "reward_change_std": 0.15456592850387096, + "reward_std": 0.4038649834692478, + "rewards/accuracy_reward": 0.06250000186264515, + "rewards/cosine_scaled_reward": -0.1466250205412507, + "step": 189 + }, + { + "clip_fraction": 0.0, + "completion_length": 3024.041717529297, + "epoch": 0.21714285714285714, + "grad_norm": 0.0596420094370842, + "kl": 2.85319983959198e-05, + "lambda_div_used": 0.5832700356841087, + "learning_rate": 1.0826776744855121e-07, + "loss": -0.0126, + "reward": -0.17394864186644554, + "reward_after_mean": -0.17394864186644554, + "reward_after_std": 0.471806388348341, + "reward_before_mean": 0.198049274738878, + "reward_before_std": 0.38090503215789795, + "reward_change_max": 0.0, + "reward_change_mean": -0.37199792452156544, + "reward_change_min": -0.5356397405266762, + "reward_change_std": 0.2117150044068694, + "reward_std": 0.4718063995242119, + "rewards/accuracy_reward": 0.22916666977107525, + "rewards/cosine_scaled_reward": -0.031117402017116547, + "step": 190 + }, + { + "clip_fraction": 0.0, + "completion_length": 2440.3541870117188, + "epoch": 0.21828571428571428, + "grad_norm": 0.0756516084074974, + "kl": 3.930646926164627e-05, + "lambda_div_used": 0.5715140625834465, + "learning_rate": 1.068365111445064e-07, + "loss": 0.0003, + "reward": -0.284542384557426, + "reward_after_mean": -0.284542384557426, + "reward_after_std": 0.447479585185647, + "reward_before_mean": 0.05547440081136301, + "reward_before_std": 0.3240885529667139, + "reward_change_max": 0.0, + "reward_change_mean": -0.34001679718494415, + "reward_change_min": -0.4750482700765133, + "reward_change_std": 0.1778492433950305, + "reward_std": 0.4474795889109373, + "rewards/accuracy_reward": 0.1666666679084301, + "rewards/cosine_scaled_reward": -0.11119226738810539, + "step": 191 + }, + { + "clip_fraction": 0.0, + "completion_length": 3487.6041870117188, + "epoch": 0.21942857142857142, + "grad_norm": 0.048071179538965225, + "kl": 3.250688314437866e-05, + "lambda_div_used": 0.5586921945214272, + "learning_rate": 1.0554024673218806e-07, + "loss": 0.0206, + "reward": -0.3871428966522217, + "reward_after_mean": -0.3871428966522217, + "reward_after_std": 0.33264124393463135, + "reward_before_mean": -0.11114806681871414, + "reward_before_std": 0.2715215114876628, + "reward_change_max": 0.0, + "reward_change_mean": -0.27599484845995903, + "reward_change_min": -0.40478406473994255, + "reward_change_std": 0.15969175938516855, + "reward_std": 0.33264124765992165, + "rewards/accuracy_reward": 0.0833333358168602, + "rewards/cosine_scaled_reward": -0.19448141008615494, + "step": 192 + }, + { + "clip_fraction": 0.0, + "completion_length": 2929.937545776367, + "epoch": 0.22057142857142858, + "grad_norm": 0.06499005109071732, + "kl": 3.759562969207764e-05, + "lambda_div_used": 0.6119573265314102, + "learning_rate": 1.0437936906629334e-07, + "loss": 0.0198, + "reward": -0.24959510192275047, + "reward_after_mean": -0.24959510192275047, + "reward_after_std": 0.5340033005923033, + "reward_before_mean": 0.009953722357749939, + "reward_before_std": 0.520604582503438, + "reward_change_max": 0.0, + "reward_change_mean": -0.25954881869256496, + "reward_change_min": -0.5182105302810669, + "reward_change_std": 0.186925214715302, + "reward_std": 0.5340033229440451, + "rewards/accuracy_reward": 0.1458333358168602, + "rewards/cosine_scaled_reward": -0.1358796115964651, + "step": 193 + }, + { + "clip_fraction": 0.0, + "completion_length": 3132.875030517578, + "epoch": 0.22171428571428572, + "grad_norm": 0.06673065572977066, + "kl": 4.443526268005371e-05, + "lambda_div_used": 0.6207702159881592, + "learning_rate": 1.0335423176140511e-07, + "loss": 0.0719, + "reward": 0.2215312235057354, + "reward_after_mean": 0.2215312235057354, + "reward_after_std": 0.6115883849561214, + "reward_before_mean": 0.7030764240771532, + "reward_before_std": 0.5582148376852274, + "reward_change_max": 0.0, + "reward_change_mean": -0.48154521360993385, + "reward_change_min": -0.7418999075889587, + "reward_change_std": 0.29459451511502266, + "reward_std": 0.611588392406702, + "rewards/accuracy_reward": 0.4583333469927311, + "rewards/cosine_scaled_reward": 0.2447430812753737, + "step": 194 + }, + { + "clip_fraction": 0.0, + "completion_length": 2998.687530517578, + "epoch": 0.22285714285714286, + "grad_norm": 0.05692190304398537, + "kl": 4.340708255767822e-05, + "lambda_div_used": 0.5588866546750069, + "learning_rate": 1.0246514708427701e-07, + "loss": -0.0321, + "reward": -0.4250563494861126, + "reward_after_mean": -0.4250563494861126, + "reward_after_std": 0.3346366826444864, + "reward_before_mean": -0.16300050355494022, + "reward_before_std": 0.26834714552387595, + "reward_change_max": 0.0, + "reward_change_mean": -0.2620558477938175, + "reward_change_min": -0.4044179916381836, + "reward_change_std": 0.15184260439127684, + "reward_std": 0.3346366863697767, + "rewards/accuracy_reward": 0.0625, + "rewards/cosine_scaled_reward": -0.22550049051642418, + "step": 195 + }, + { + "clip_fraction": 0.0, + "completion_length": 3551.6041870117188, + "epoch": 0.224, + "grad_norm": 0.051314327865839005, + "kl": 4.8547983169555664e-05, + "lambda_div_used": 0.5673639252781868, + "learning_rate": 1.017123858587145e-07, + "loss": 0.0144, + "reward": -0.43661339208483696, + "reward_after_mean": -0.43661339208483696, + "reward_after_std": 0.3381017968058586, + "reward_before_mean": -0.18068170547485352, + "reward_before_std": 0.3060085875913501, + "reward_change_max": 0.0, + "reward_change_mean": -0.25593167915940285, + "reward_change_min": -0.436182364821434, + "reward_change_std": 0.16187659837305546, + "reward_std": 0.3381018117070198, + "rewards/accuracy_reward": 0.0625, + "rewards/cosine_scaled_reward": -0.24318172316998243, + "step": 196 + }, + { + "clip_fraction": 0.0, + "completion_length": 2789.125068664551, + "epoch": 0.22514285714285714, + "grad_norm": 0.08223231136798859, + "kl": 3.402773290872574e-05, + "lambda_div_used": 0.6149426028132439, + "learning_rate": 1.0109617738307911e-07, + "loss": 0.0526, + "reward": 0.08269345387816429, + "reward_after_mean": 0.08269345387816429, + "reward_after_std": 0.5922380294650793, + "reward_before_mean": 0.4997589197009802, + "reward_before_std": 0.5276397680863738, + "reward_change_max": 0.0, + "reward_change_mean": -0.4170654658228159, + "reward_change_min": -0.6376415118575096, + "reward_change_std": 0.24999654106795788, + "reward_std": 0.592238049954176, + "rewards/accuracy_reward": 0.39583334140479565, + "rewards/cosine_scaled_reward": 0.10392560251057148, + "step": 197 + }, + { + "clip_fraction": 0.0, + "completion_length": 2770.2708435058594, + "epoch": 0.22628571428571428, + "grad_norm": 0.07601747661828995, + "kl": 3.542006015777588e-05, + "lambda_div_used": 0.5800714045763016, + "learning_rate": 1.0061670936044178e-07, + "loss": -0.0178, + "reward": -0.24220025539398193, + "reward_after_mean": -0.24220025539398193, + "reward_after_std": 0.4058863651007414, + "reward_before_mean": 0.06675046496093273, + "reward_before_std": 0.36683204025030136, + "reward_change_max": 0.0, + "reward_change_mean": -0.30895073898136616, + "reward_change_min": -0.4967211000621319, + "reward_change_std": 0.18911676667630672, + "reward_std": 0.40588637441396713, + "rewards/accuracy_reward": 0.1666666716337204, + "rewards/cosine_scaled_reward": -0.09991620294749737, + "step": 198 + }, + { + "clip_fraction": 0.0, + "completion_length": 3554.0416870117188, + "epoch": 0.22742857142857142, + "grad_norm": 0.04930044710636139, + "kl": 4.279613494873047e-05, + "lambda_div_used": 0.5560082867741585, + "learning_rate": 1.002741278414069e-07, + "loss": 0.0027, + "reward": -0.4229874052107334, + "reward_after_mean": -0.4229874052107334, + "reward_after_std": 0.31992789916694164, + "reward_before_mean": -0.14317837683483958, + "reward_before_std": 0.25433824164792895, + "reward_change_max": 0.0, + "reward_change_mean": -0.27980900928378105, + "reward_change_min": -0.4250169135630131, + "reward_change_std": 0.16277197189629078, + "reward_std": 0.31992790661752224, + "rewards/accuracy_reward": 0.0625, + "rewards/cosine_scaled_reward": -0.20567839220166206, + "step": 199 + }, + { + "clip_fraction": 0.0, + "completion_length": 2299.5209045410156, + "epoch": 0.22857142857142856, + "grad_norm": 0.0840829536318779, + "kl": 4.097074270248413e-05, + "lambda_div_used": 0.6226711273193359, + "learning_rate": 1.0006853717962393e-07, + "loss": 0.0203, + "reward": 0.062016794458031654, + "reward_after_mean": 0.062016794458031654, + "reward_after_std": 0.631916331127286, + "reward_before_mean": 0.4668920338153839, + "reward_before_std": 0.5674512181431055, + "reward_change_max": 0.0, + "reward_change_mean": -0.40487524308264256, + "reward_change_min": -0.6527138948440552, + "reward_change_std": 0.250981617718935, + "reward_std": 0.6319163534790277, + "rewards/accuracy_reward": 0.35416667349636555, + "rewards/cosine_scaled_reward": 0.11272535985335708, + "step": 200 + }, + { + "clip_fraction": 0.0, + "completion_length": 2760.7084197998047, + "epoch": 0.2297142857142857, + "grad_norm": 0.06477575749158859, + "kl": 1.7711427062749863e-05, + "lambda_div_used": 0.6429438292980194, + "learning_rate": 1e-07, + "loss": -0.0022, + "reward": 0.15019571036100388, + "reward_after_mean": 0.15019571036100388, + "reward_after_std": 0.7328120246529579, + "reward_before_mean": 0.5535202682949603, + "reward_before_std": 0.6601617820560932, + "reward_change_max": 0.0, + "reward_change_mean": -0.4033245462924242, + "reward_change_min": -0.6303750872612, + "reward_change_std": 0.24585676938295364, + "reward_std": 0.7328120358288288, + "rewards/accuracy_reward": 0.41666667349636555, + "rewards/cosine_scaled_reward": 0.13685358315706253, + "step": 201 + }, + { + "clip_fraction": 0.0, + "completion_length": 2511.937511444092, + "epoch": 0.23085714285714284, + "grad_norm": 0.08345558494329453, + "kl": 4.0203332901000977e-05, + "lambda_div_used": 0.5811712816357613, + "learning_rate": 7.72273839962904e-07, + "loss": 0.0587, + "reward": 0.03902309015393257, + "reward_after_mean": 0.03902309015393257, + "reward_after_std": 0.5094954669475555, + "reward_before_mean": 0.5408022310584784, + "reward_before_std": 0.37269798293709755, + "reward_change_max": 0.0, + "reward_change_mean": -0.5017791502177715, + "reward_change_min": -0.7236880213022232, + "reward_change_std": 0.28125342447310686, + "reward_std": 0.5094954781234264, + "rewards/accuracy_reward": 0.3958333395421505, + "rewards/cosine_scaled_reward": 0.14496888127177954, + "step": 202 + }, + { + "clip_fraction": 0.0, + "completion_length": 3065.3333435058594, + "epoch": 0.232, + "grad_norm": 0.05931547284126282, + "kl": 5.182623863220215e-05, + "lambda_div_used": 0.5537669658660889, + "learning_rate": 7.695368466124296e-07, + "loss": 0.0447, + "reward": -0.3224771413952112, + "reward_after_mean": -0.3224771413952112, + "reward_after_std": 0.36307925172150135, + "reward_before_mean": 0.029698201455175877, + "reward_before_std": 0.23980092909187078, + "reward_change_max": 0.0, + "reward_change_mean": -0.3521753493696451, + "reward_change_min": -0.49568963050842285, + "reward_change_std": 0.18385440576821566, + "reward_std": 0.36307926289737225, + "rewards/accuracy_reward": 0.14583333395421505, + "rewards/cosine_scaled_reward": -0.1161351166665554, + "step": 203 + }, + { + "clip_fraction": 0.0, + "completion_length": 2511.833366394043, + "epoch": 0.23314285714285715, + "grad_norm": 0.07267706841230392, + "kl": 5.4508447647094727e-05, + "lambda_div_used": 0.5887176766991615, + "learning_rate": 7.667891533457718e-07, + "loss": 0.0023, + "reward": -0.23160922899842262, + "reward_after_mean": -0.23160922899842262, + "reward_after_std": 0.430733734741807, + "reward_before_mean": 0.06992286071181297, + "reward_before_std": 0.4116028640419245, + "reward_change_max": 0.0, + "reward_change_mean": -0.301532082259655, + "reward_change_min": -0.49793890863657, + "reward_change_std": 0.19511268101632595, + "reward_std": 0.43073374405503273, + "rewards/accuracy_reward": 0.1458333358168602, + "rewards/cosine_scaled_reward": -0.07591048628091812, + "step": 204 + }, + { + "clip_fraction": 0.0, + "completion_length": 2967.1875915527344, + "epoch": 0.2342857142857143, + "grad_norm": 0.06776981800794601, + "kl": 4.439055919647217e-05, + "lambda_div_used": 0.653583250939846, + "learning_rate": 7.640308940816239e-07, + "loss": 0.1014, + "reward": -0.08090712130069733, + "reward_after_mean": -0.08090712130069733, + "reward_after_std": 0.7262772191315889, + "reward_before_mean": 0.17835846357047558, + "reward_before_std": 0.7181071005761623, + "reward_change_max": 0.0, + "reward_change_mean": -0.2592655848711729, + "reward_change_min": -0.4743462074548006, + "reward_change_std": 0.18475584778934717, + "reward_std": 0.7262772284448147, + "rewards/accuracy_reward": 0.2291666716337204, + "rewards/cosine_scaled_reward": -0.05080821271985769, + "step": 205 + }, + { + "clip_fraction": 0.0, + "completion_length": 3085.5208892822266, + "epoch": 0.23542857142857143, + "grad_norm": 0.06689873337745667, + "kl": 3.569573163986206e-05, + "lambda_div_used": 0.5862618833780289, + "learning_rate": 7.612622032536507e-07, + "loss": 0.0353, + "reward": -0.28365214727818966, + "reward_after_mean": -0.28365214727818966, + "reward_after_std": 0.44012872874736786, + "reward_before_mean": 0.004427256062626839, + "reward_before_std": 0.3988625044003129, + "reward_change_max": 0.0, + "reward_change_mean": -0.2880793921649456, + "reward_change_min": -0.49235254526138306, + "reward_change_std": 0.18395678792148829, + "reward_std": 0.44012873619794846, + "rewards/accuracy_reward": 0.1250000037252903, + "rewards/cosine_scaled_reward": -0.12057275045663118, + "step": 206 + }, + { + "clip_fraction": 0.0, + "completion_length": 3146.6458740234375, + "epoch": 0.23657142857142857, + "grad_norm": 0.06122450903058052, + "kl": 3.725104033946991e-05, + "lambda_div_used": 0.579835832118988, + "learning_rate": 7.584832158039378e-07, + "loss": 0.0182, + "reward": -0.28688727831467986, + "reward_after_mean": -0.28688727831467986, + "reward_after_std": 0.4150819983333349, + "reward_before_mean": 0.009919969365000725, + "reward_before_std": 0.3629662115126848, + "reward_change_max": 0.0, + "reward_change_mean": -0.29680725932121277, + "reward_change_min": -0.45919613167643547, + "reward_change_std": 0.17279880121350288, + "reward_std": 0.41508200392127037, + "rewards/accuracy_reward": 0.12500000558793545, + "rewards/cosine_scaled_reward": -0.11508002015762031, + "step": 207 + }, + { + "clip_fraction": 0.0, + "completion_length": 2780.291717529297, + "epoch": 0.2377142857142857, + "grad_norm": 0.06370716542005539, + "kl": 3.692507743835449e-05, + "lambda_div_used": 0.5807016789913177, + "learning_rate": 7.556940671764124e-07, + "loss": 0.0403, + "reward": -0.17094913870096207, + "reward_after_mean": -0.17094913870096207, + "reward_after_std": 0.46442117914557457, + "reward_before_mean": 0.20349296741187572, + "reward_before_std": 0.3671952560544014, + "reward_change_max": 0.0, + "reward_change_mean": -0.3744421415030956, + "reward_change_min": -0.5505415536463261, + "reward_change_std": 0.21210224367678165, + "reward_std": 0.46442119032144547, + "rewards/accuracy_reward": 0.20833333395421505, + "rewards/cosine_scaled_reward": -0.004840357229113579, + "step": 208 + }, + { + "clip_fraction": 0.0, + "completion_length": 2574.6042079925537, + "epoch": 0.23885714285714285, + "grad_norm": 0.08756011724472046, + "kl": 1.8547754734754562e-05, + "lambda_div_used": 0.6246868968009949, + "learning_rate": 7.528948933102438e-07, + "loss": 0.0837, + "reward": 0.028150439262390137, + "reward_after_mean": 0.028150439262390137, + "reward_after_std": 0.5636487938463688, + "reward_before_mean": 0.3838757500052452, + "reward_before_std": 0.576031070202589, + "reward_change_max": 0.0, + "reward_change_mean": -0.3557253200560808, + "reward_change_min": -0.5899170190095901, + "reward_change_std": 0.23946599010378122, + "reward_std": 0.5636488180607557, + "rewards/accuracy_reward": 0.3333333469927311, + "rewards/cosine_scaled_reward": 0.05054241791367531, + "step": 209 + }, + { + "clip_fraction": 0.0, + "completion_length": 2779.3333435058594, + "epoch": 0.24, + "grad_norm": 0.06111512333154678, + "kl": 3.0368566513061523e-05, + "lambda_div_used": 0.5897415727376938, + "learning_rate": 7.500858306332172e-07, + "loss": 0.0277, + "reward": -0.25500940857455134, + "reward_after_mean": -0.25500940857455134, + "reward_after_std": 0.47190882451832294, + "reward_before_mean": 0.03478116978658363, + "reward_before_std": 0.4098346810787916, + "reward_change_max": 0.0, + "reward_change_mean": -0.2897905595600605, + "reward_change_min": -0.42270801588892937, + "reward_change_std": 0.1620404813438654, + "reward_std": 0.4719088301062584, + "rewards/accuracy_reward": 0.1458333395421505, + "rewards/cosine_scaled_reward": -0.11105216934811324, + "step": 210 + }, + { + "clip_fraction": 0.0, + "completion_length": 2558.8541946411133, + "epoch": 0.24114285714285713, + "grad_norm": 0.06923027336597443, + "kl": 3.6522746086120605e-05, + "lambda_div_used": 0.5664671063423157, + "learning_rate": 7.472670160550848e-07, + "loss": 0.0729, + "reward": -0.06898702308535576, + "reward_after_mean": -0.06898702308535576, + "reward_after_std": 0.4354093614965677, + "reward_before_mean": 0.4075740482658148, + "reward_before_std": 0.30287738889455795, + "reward_change_max": 0.0, + "reward_change_mean": -0.4765610620379448, + "reward_change_min": -0.6883851811289787, + "reward_change_std": 0.262312775477767, + "reward_std": 0.4354093801230192, + "rewards/accuracy_reward": 0.3125, + "rewards/cosine_scaled_reward": 0.09507404454052448, + "step": 211 + }, + { + "clip_fraction": 0.0, + "completion_length": 2327.750015258789, + "epoch": 0.2422857142857143, + "grad_norm": 0.08028864860534668, + "kl": 2.8003007173538208e-05, + "lambda_div_used": 0.5712975114583969, + "learning_rate": 7.444385869608921e-07, + "loss": -0.0113, + "reward": -0.09517045877873898, + "reward_after_mean": -0.09517045877873898, + "reward_after_std": 0.49211281538009644, + "reward_before_mean": 0.3566615767776966, + "reward_before_std": 0.32028379291296005, + "reward_change_max": 0.0, + "reward_change_mean": -0.4518320318311453, + "reward_change_min": -0.6186705827713013, + "reward_change_std": 0.23167487233877182, + "reward_std": 0.4921128321439028, + "rewards/accuracy_reward": 0.2916666679084301, + "rewards/cosine_scaled_reward": 0.06499490264104679, + "step": 212 + }, + { + "clip_fraction": 0.0, + "completion_length": 2145.6041946411133, + "epoch": 0.24342857142857144, + "grad_norm": 0.09073800593614578, + "kl": 4.177819937467575e-05, + "lambda_div_used": 0.6168783828616142, + "learning_rate": 7.416006812042827e-07, + "loss": 0.027, + "reward": -0.08367926510982215, + "reward_after_mean": -0.08367926510982215, + "reward_after_std": 0.5790203902870417, + "reward_before_mean": 0.24448610469698906, + "reward_before_std": 0.5364782512187958, + "reward_change_max": 0.0, + "reward_change_mean": -0.32816537097096443, + "reward_change_min": -0.5068525895476341, + "reward_change_std": 0.19713077135384083, + "reward_std": 0.579020407050848, + "rewards/accuracy_reward": 0.2083333395421505, + "rewards/cosine_scaled_reward": 0.036152773071080446, + "step": 213 + }, + { + "clip_fraction": 0.0, + "completion_length": 2982.6458587646484, + "epoch": 0.24457142857142858, + "grad_norm": 0.06497277319431305, + "kl": 3.772228956222534e-05, + "lambda_div_used": 0.6168569102883339, + "learning_rate": 7.387534371007797e-07, + "loss": 0.0247, + "reward": -0.10639690980315208, + "reward_after_mean": -0.10639690980315208, + "reward_after_std": 0.563305439427495, + "reward_before_mean": 0.21648684330284595, + "reward_before_std": 0.5375220291316509, + "reward_change_max": 0.0, + "reward_change_mean": -0.3228837437927723, + "reward_change_min": -0.5641361065208912, + "reward_change_std": 0.20987980626523495, + "reward_std": 0.5633054543286562, + "rewards/accuracy_reward": 0.20833334140479565, + "rewards/cosine_scaled_reward": 0.00815348862670362, + "step": 214 + }, + { + "clip_fraction": 0.0, + "completion_length": 2382.2500610351562, + "epoch": 0.24571428571428572, + "grad_norm": 0.07877103984355927, + "kl": 4.485622048377991e-05, + "lambda_div_used": 0.5695274397730827, + "learning_rate": 7.358969934210438e-07, + "loss": 0.0351, + "reward": -0.39196348818950355, + "reward_after_mean": -0.39196348818950355, + "reward_after_std": 0.39862883277237415, + "reward_before_mean": -0.128106027841568, + "reward_before_std": 0.31178954988718033, + "reward_change_max": 0.0, + "reward_change_mean": -0.263857439160347, + "reward_change_min": -0.38021961972117424, + "reward_change_std": 0.13994430005550385, + "reward_std": 0.39862884022295475, + "rewards/accuracy_reward": 0.0416666679084301, + "rewards/cosine_scaled_reward": -0.1697727027349174, + "step": 215 + }, + { + "clip_fraction": 0.0, + "completion_length": 2156.3958740234375, + "epoch": 0.24685714285714286, + "grad_norm": 0.11318857222795486, + "kl": 4.297494888305664e-05, + "lambda_div_used": 0.5965342745184898, + "learning_rate": 7.330314893841101e-07, + "loss": 0.0186, + "reward": -0.22045887634158134, + "reward_after_mean": -0.22045887634158134, + "reward_after_std": 0.4805217906832695, + "reward_before_mean": 0.08258800266776234, + "reward_before_std": 0.4415160808712244, + "reward_change_max": 0.0, + "reward_change_mean": -0.3030468635261059, + "reward_change_min": -0.4999500457197428, + "reward_change_std": 0.185057258233428, + "reward_std": 0.4805217981338501, + "rewards/accuracy_reward": 0.16666667349636555, + "rewards/cosine_scaled_reward": -0.08407866954803467, + "step": 216 + }, + { + "clip_fraction": 0.0, + "completion_length": 2763.520866394043, + "epoch": 0.248, + "grad_norm": 0.067295141518116, + "kl": 3.0644237995147705e-05, + "lambda_div_used": 0.6201670467853546, + "learning_rate": 7.301570646506027e-07, + "loss": 0.0139, + "reward": -0.19884846359491348, + "reward_after_mean": -0.19884846359491348, + "reward_after_std": 0.5856006741523743, + "reward_before_mean": 0.06158541014883667, + "reward_before_std": 0.5545480605214834, + "reward_change_max": 0.0, + "reward_change_mean": -0.2604338899254799, + "reward_change_min": -0.503634799271822, + "reward_change_std": 0.1784855630248785, + "reward_std": 0.5856006946414709, + "rewards/accuracy_reward": 0.16666666977107525, + "rewards/cosine_scaled_reward": -0.10508125275373459, + "step": 217 + }, + { + "clip_fraction": 0.0, + "completion_length": 2880.7083587646484, + "epoch": 0.24914285714285714, + "grad_norm": 0.07706139236688614, + "kl": 3.7044286727905273e-05, + "lambda_div_used": 0.6074161231517792, + "learning_rate": 7.27273859315928e-07, + "loss": -0.0223, + "reward": -0.19500153325498104, + "reward_after_mean": -0.19500153325498104, + "reward_after_std": 0.5332945492118597, + "reward_before_mean": 0.08328226953744888, + "reward_before_std": 0.5006468072533607, + "reward_change_max": 0.0, + "reward_change_mean": -0.27828381210565567, + "reward_change_min": -0.5113113783299923, + "reward_change_std": 0.18696410488337278, + "reward_std": 0.5332945715636015, + "rewards/accuracy_reward": 0.1666666716337204, + "rewards/cosine_scaled_reward": -0.08338439278304577, + "step": 218 + }, + { + "clip_fraction": 0.0, + "completion_length": 2352.645854949951, + "epoch": 0.2502857142857143, + "grad_norm": 0.10948827862739563, + "kl": 3.1441450119018555e-05, + "lambda_div_used": 0.5561031624674797, + "learning_rate": 7.243820139034464e-07, + "loss": -0.0096, + "reward": -0.029067307710647583, + "reward_after_mean": -0.029067307710647583, + "reward_after_std": 0.41141366213560104, + "reward_before_mean": 0.4911606255918741, + "reward_before_std": 0.2519808644428849, + "reward_change_max": 0.0, + "reward_change_mean": -0.5202279426157475, + "reward_change_min": -0.7093416638672352, + "reward_change_std": 0.27748389169573784, + "reward_std": 0.41141367703676224, + "rewards/accuracy_reward": 0.3541666716337204, + "rewards/cosine_scaled_reward": 0.13699393905699253, + "step": 219 + }, + { + "clip_fraction": 0.0, + "completion_length": 2635.5833435058594, + "epoch": 0.25142857142857145, + "grad_norm": 0.08914919197559357, + "kl": 3.5672448575496674e-05, + "lambda_div_used": 0.5308618620038033, + "learning_rate": 7.214816693576234e-07, + "loss": -0.0362, + "reward": -0.5146691724658012, + "reward_after_mean": -0.5146691724658012, + "reward_after_std": 0.23347610421478748, + "reward_before_mean": -0.2369950506836176, + "reward_before_std": 0.13545648753643036, + "reward_change_max": 0.0, + "reward_change_mean": -0.27767411433160305, + "reward_change_min": -0.3908136747777462, + "reward_change_std": 0.14405533485114574, + "reward_std": 0.23347610607743263, + "rewards/accuracy_reward": 0.0, + "rewards/cosine_scaled_reward": -0.2369950469583273, + "step": 220 + }, + { + "clip_fraction": 0.0, + "completion_length": 2279.562530517578, + "epoch": 0.25257142857142856, + "grad_norm": 0.0954662561416626, + "kl": 2.4788081645965576e-05, + "lambda_div_used": 0.5764833092689514, + "learning_rate": 7.185729670371604e-07, + "loss": -0.0168, + "reward": 0.023251693695783615, + "reward_after_mean": 0.023251693695783615, + "reward_after_std": 0.4863298423588276, + "reward_before_mean": 0.5212067291140556, + "reward_before_std": 0.34889572812244296, + "reward_change_max": 0.0, + "reward_change_mean": -0.49795500561594963, + "reward_change_min": -0.7016883753240108, + "reward_change_std": 0.27911104913800955, + "reward_std": 0.48632985167205334, + "rewards/accuracy_reward": 0.4166666716337204, + "rewards/cosine_scaled_reward": 0.10454002395272255, + "step": 221 + }, + { + "clip_fraction": 0.0, + "completion_length": 2104.9583740234375, + "epoch": 0.2537142857142857, + "grad_norm": 0.0776790902018547, + "kl": 2.4726614356040955e-05, + "lambda_div_used": 0.5997953563928604, + "learning_rate": 7.156560487081051e-07, + "loss": -0.0148, + "reward": -0.04329463094472885, + "reward_after_mean": -0.04329463094472885, + "reward_after_std": 0.48106030002236366, + "reward_before_mean": 0.33003126084804535, + "reward_before_std": 0.46051184553653, + "reward_change_max": 0.0, + "reward_change_mean": -0.3733258917927742, + "reward_change_min": -0.5728014186024666, + "reward_change_std": 0.23287776950746775, + "reward_std": 0.48106031119823456, + "rewards/accuracy_reward": 0.2708333432674408, + "rewards/cosine_scaled_reward": 0.0591979268938303, + "step": 222 + }, + { + "clip_fraction": 0.0, + "completion_length": 2521.3333587646484, + "epoch": 0.25485714285714284, + "grad_norm": 0.07036174088716507, + "kl": 2.9304996132850647e-05, + "lambda_div_used": 0.5750112235546112, + "learning_rate": 7.127310565369415e-07, + "loss": 0.0216, + "reward": -0.1473498847335577, + "reward_after_mean": -0.1473498847335577, + "reward_after_std": 0.4359878208488226, + "reward_before_mean": 0.24288302287459373, + "reward_before_std": 0.3438769578933716, + "reward_change_max": 0.0, + "reward_change_mean": -0.39023288898169994, + "reward_change_min": -0.588653527200222, + "reward_change_std": 0.22452317085117102, + "reward_std": 0.43598783388733864, + "rewards/accuracy_reward": 0.2708333395421505, + "rewards/cosine_scaled_reward": -0.027950339019298553, + "step": 223 + }, + { + "clip_fraction": 0.0, + "completion_length": 3317.729217529297, + "epoch": 0.256, + "grad_norm": 0.05309538170695305, + "kl": 2.8050970286130905e-05, + "lambda_div_used": 0.5725216493010521, + "learning_rate": 7.097981330836616e-07, + "loss": 0.069, + "reward": -0.30242439545691013, + "reward_after_mean": -0.30242439545691013, + "reward_after_std": 0.38869454339146614, + "reward_before_mean": -0.000308917835354805, + "reward_before_std": 0.3251040354371071, + "reward_change_max": 0.0, + "reward_change_mean": -0.30211549811065197, + "reward_change_min": -0.4553787522017956, + "reward_change_std": 0.171081081032753, + "reward_std": 0.3886945564299822, + "rewards/accuracy_reward": 0.12500000558793545, + "rewards/cosine_scaled_reward": -0.12530891119968146, + "step": 224 + }, + { + "clip_fraction": 0.0, + "completion_length": 3037.208335876465, + "epoch": 0.2571428571428571, + "grad_norm": 0.0768936425447464, + "kl": 4.080682992935181e-05, + "lambda_div_used": 0.5907088667154312, + "learning_rate": 7.068574212948169e-07, + "loss": 0.0169, + "reward": -0.28160014655441046, + "reward_after_mean": -0.28160014655441046, + "reward_after_std": 0.48472702503204346, + "reward_before_mean": -0.008234186680056155, + "reward_before_std": 0.41540220472961664, + "reward_change_max": 0.0, + "reward_change_mean": -0.27336596697568893, + "reward_change_min": -0.4061685614287853, + "reward_change_std": 0.15673903841525316, + "reward_std": 0.48472702503204346, + "rewards/accuracy_reward": 0.1250000037252903, + "rewards/cosine_scaled_reward": -0.13323417864739895, + "step": 225 + }, + { + "clip_fraction": 0.0, + "completion_length": 2861.416702270508, + "epoch": 0.2582857142857143, + "grad_norm": 0.07499676197767258, + "kl": 3.580749034881592e-05, + "lambda_div_used": 0.6208535805344582, + "learning_rate": 7.039090644965509e-07, + "loss": 0.0702, + "reward": -0.08291551470756531, + "reward_after_mean": -0.08291551470756531, + "reward_after_std": 0.5764818880707026, + "reward_before_mean": 0.23111886344850063, + "reward_before_std": 0.5593543313443661, + "reward_change_max": 0.0, + "reward_change_mean": -0.3140343725681305, + "reward_change_min": -0.5283008627593517, + "reward_change_std": 0.20683539099991322, + "reward_std": 0.5764818955212831, + "rewards/accuracy_reward": 0.27083334140479565, + "rewards/cosine_scaled_reward": -0.03971448230731767, + "step": 226 + }, + { + "clip_fraction": 0.0, + "completion_length": 1926.6875305175781, + "epoch": 0.25942857142857145, + "grad_norm": 0.10923109948635101, + "kl": 4.176795482635498e-05, + "lambda_div_used": 0.5834182798862457, + "learning_rate": 7.009532063876148e-07, + "loss": -0.0447, + "reward": -0.2778073139488697, + "reward_after_mean": -0.2778073139488697, + "reward_after_std": 0.4395454227924347, + "reward_before_mean": 0.009000460617244244, + "reward_before_std": 0.3901812704280019, + "reward_change_max": 0.0, + "reward_change_mean": -0.28680778108537197, + "reward_change_min": -0.45677450299263, + "reward_change_std": 0.18111994117498398, + "reward_std": 0.43954543210566044, + "rewards/accuracy_reward": 0.125, + "rewards/cosine_scaled_reward": -0.1159995449706912, + "step": 227 + }, + { + "clip_fraction": 0.0, + "completion_length": 2502.562545776367, + "epoch": 0.26057142857142856, + "grad_norm": 0.08182154595851898, + "kl": 1.8984079360961914e-05, + "lambda_div_used": 0.6025907471776009, + "learning_rate": 6.979899910323624e-07, + "loss": 0.074, + "reward": 0.03976001590490341, + "reward_after_mean": 0.03976001590490341, + "reward_after_std": 0.5421929359436035, + "reward_before_mean": 0.4711545445024967, + "reward_before_std": 0.4750876808539033, + "reward_change_max": 0.0, + "reward_change_mean": -0.4313945174217224, + "reward_change_min": -0.6774051003158092, + "reward_change_std": 0.2637105621397495, + "reward_std": 0.5421929433941841, + "rewards/accuracy_reward": 0.3541666753590107, + "rewards/cosine_scaled_reward": 0.11698784306645393, + "step": 228 + }, + { + "clip_fraction": 0.0, + "completion_length": 3196.3125228881836, + "epoch": 0.26171428571428573, + "grad_norm": 0.06398586928844452, + "kl": 4.0434300899505615e-05, + "lambda_div_used": 0.5669709742069244, + "learning_rate": 6.950195628537299e-07, + "loss": 0.0135, + "reward": -0.15348458290100098, + "reward_after_mean": -0.15348458290100098, + "reward_after_std": 0.42672324273735285, + "reward_before_mean": 0.26130594592541456, + "reward_before_std": 0.3047938751988113, + "reward_change_max": 0.0, + "reward_change_mean": -0.41479056514799595, + "reward_change_min": -0.5674791261553764, + "reward_change_std": 0.22689451277256012, + "reward_std": 0.4267232706770301, + "rewards/accuracy_reward": 0.22916666977107525, + "rewards/cosine_scaled_reward": 0.03213928057812154, + "step": 229 + }, + { + "clip_fraction": 0.0, + "completion_length": 3100.5000228881836, + "epoch": 0.26285714285714284, + "grad_norm": 0.05996527522802353, + "kl": 2.22623348236084e-05, + "lambda_div_used": 0.5811162814497948, + "learning_rate": 6.920420666261961e-07, + "loss": 0.0391, + "reward": -0.36068666726350784, + "reward_after_mean": -0.36068666726350784, + "reward_after_std": 0.43734684213995934, + "reward_before_mean": -0.08804995659738779, + "reward_before_std": 0.37305002473294735, + "reward_change_max": 0.0, + "reward_change_mean": -0.2726367134600878, + "reward_change_min": -0.4353605732321739, + "reward_change_std": 0.16089125256985426, + "reward_std": 0.43734684586524963, + "rewards/accuracy_reward": 0.06250000186264515, + "rewards/cosine_scaled_reward": -0.15054996183607727, + "step": 230 + }, + { + "clip_fraction": 0.0, + "completion_length": 2678.7083587646484, + "epoch": 0.264, + "grad_norm": 0.06770680844783783, + "kl": 2.5890767574310303e-05, + "lambda_div_used": 0.5817164853215218, + "learning_rate": 6.890576474687263e-07, + "loss": 0.0722, + "reward": -0.13625335786491632, + "reward_after_mean": -0.13625335786491632, + "reward_after_std": 0.46202925965189934, + "reward_before_mean": 0.25235490314662457, + "reward_before_std": 0.37311020120978355, + "reward_change_max": 0.0, + "reward_change_mean": -0.3886082824319601, + "reward_change_min": -0.6004670634865761, + "reward_change_std": 0.2275555245578289, + "reward_std": 0.4620292726904154, + "rewards/accuracy_reward": 0.2708333395421505, + "rewards/cosine_scaled_reward": -0.018478410318493843, + "step": 231 + }, + { + "clip_fraction": 0.0, + "completion_length": 3258.625030517578, + "epoch": 0.2651428571428571, + "grad_norm": 0.05468503385782242, + "kl": 3.515183925628662e-05, + "lambda_div_used": 0.5800251811742783, + "learning_rate": 6.860664508377001e-07, + "loss": 0.0159, + "reward": -0.3428461756557226, + "reward_after_mean": -0.3428461756557226, + "reward_after_std": 0.4266065489500761, + "reward_before_mean": -0.07336848601698875, + "reward_before_std": 0.36755594704300165, + "reward_change_max": 0.0, + "reward_change_mean": -0.26947769708931446, + "reward_change_min": -0.43084757775068283, + "reward_change_std": 0.1597052849829197, + "reward_std": 0.4266065787523985, + "rewards/accuracy_reward": 0.08333333395421505, + "rewards/cosine_scaled_reward": -0.1567018236964941, + "step": 232 + }, + { + "clip_fraction": 0.0, + "completion_length": 2661.2292251586914, + "epoch": 0.2662857142857143, + "grad_norm": 0.08775883167982101, + "kl": 3.3989548683166504e-05, + "lambda_div_used": 0.6155472174286842, + "learning_rate": 6.83068622519821e-07, + "loss": -0.016, + "reward": -0.22876367531716824, + "reward_after_mean": -0.22876367531716824, + "reward_after_std": 0.5766322333365679, + "reward_before_mean": 0.027781556826084852, + "reward_before_std": 0.5290827043354511, + "reward_change_max": 0.0, + "reward_change_mean": -0.2565452288836241, + "reward_change_min": -0.4581022933125496, + "reward_change_std": 0.16627201065421104, + "reward_std": 0.5766322445124388, + "rewards/accuracy_reward": 0.1458333358168602, + "rewards/cosine_scaled_reward": -0.11805178504437208, + "step": 233 + }, + { + "clip_fraction": 0.0, + "completion_length": 2694.3125228881836, + "epoch": 0.2674285714285714, + "grad_norm": 0.11633959412574768, + "kl": 3.643333911895752e-05, + "lambda_div_used": 0.5539941042661667, + "learning_rate": 6.800643086250121e-07, + "loss": -0.0056, + "reward": -0.22800572216510773, + "reward_after_mean": -0.22800572216510773, + "reward_after_std": 0.3484720904380083, + "reward_before_mean": 0.16009100899100304, + "reward_before_std": 0.24206165876239538, + "reward_change_max": 0.0, + "reward_change_mean": -0.3880967255681753, + "reward_change_min": -0.5548957660794258, + "reward_change_std": 0.21049270872026682, + "reward_std": 0.34847209975123405, + "rewards/accuracy_reward": 0.2291666716337204, + "rewards/cosine_scaled_reward": -0.06907567009329796, + "step": 234 + }, + { + "clip_fraction": 0.0, + "completion_length": 2490.4375, + "epoch": 0.26857142857142857, + "grad_norm": 0.10378725826740265, + "kl": 2.8094742447137833e-05, + "lambda_div_used": 0.5782083421945572, + "learning_rate": 6.770536555792944e-07, + "loss": -0.0425, + "reward": 0.01590564101934433, + "reward_after_mean": 0.01590564101934433, + "reward_after_std": 0.5053408965468407, + "reward_before_mean": 0.49857149738818407, + "reward_before_std": 0.3586234971880913, + "reward_change_max": 0.0, + "reward_change_mean": -0.48266585171222687, + "reward_change_min": -0.7199156694114208, + "reward_change_std": 0.2697788691148162, + "reward_std": 0.505340900272131, + "rewards/accuracy_reward": 0.3750000037252903, + "rewards/cosine_scaled_reward": 0.12357146013528109, + "step": 235 + }, + { + "clip_fraction": 0.0, + "completion_length": 2824.7917098999023, + "epoch": 0.26971428571428574, + "grad_norm": 0.06474865972995758, + "kl": 2.1502375602722168e-05, + "lambda_div_used": 0.5763456672430038, + "learning_rate": 6.740368101176495e-07, + "loss": 0.0011, + "reward": -0.3977004513144493, + "reward_after_mean": -0.3977004513144493, + "reward_after_std": 0.41489073634147644, + "reward_before_mean": -0.1460555698722601, + "reward_before_std": 0.347917802631855, + "reward_change_max": 0.0, + "reward_change_mean": -0.2516448702663183, + "reward_change_min": -0.42327918112277985, + "reward_change_std": 0.14836463797837496, + "reward_std": 0.4148907568305731, + "rewards/accuracy_reward": 0.06250000186264515, + "rewards/cosine_scaled_reward": -0.20855557406321168, + "step": 236 + }, + { + "clip_fraction": 0.0, + "completion_length": 2764.9583740234375, + "epoch": 0.27085714285714285, + "grad_norm": 0.06224860996007919, + "kl": 2.5756657123565674e-05, + "lambda_div_used": 0.5886275842785835, + "learning_rate": 6.710139192768694e-07, + "loss": -0.0228, + "reward": -0.1175742200575769, + "reward_after_mean": -0.1175742200575769, + "reward_after_std": 0.4582329224795103, + "reward_before_mean": 0.24255024455487728, + "reward_before_std": 0.40142686292529106, + "reward_change_max": 0.0, + "reward_change_mean": -0.36012447625398636, + "reward_change_min": -0.5257687419652939, + "reward_change_std": 0.20567627251148224, + "reward_std": 0.4582329299300909, + "rewards/accuracy_reward": 0.22916667722165585, + "rewards/cosine_scaled_reward": 0.013383567042183131, + "step": 237 + }, + { + "clip_fraction": 0.0, + "completion_length": 3222.479217529297, + "epoch": 0.272, + "grad_norm": 0.05841728672385216, + "kl": 3.546103835105896e-05, + "lambda_div_used": 0.6281886473298073, + "learning_rate": 6.679851303883891e-07, + "loss": 0.0655, + "reward": 0.044486068189144135, + "reward_after_mean": 0.044486068189144135, + "reward_after_std": 0.6025157757103443, + "reward_before_mean": 0.4036002438515425, + "reward_before_std": 0.6015807576477528, + "reward_change_max": 0.0, + "reward_change_mean": -0.359114158898592, + "reward_change_min": -0.609935961663723, + "reward_change_std": 0.24452881701290607, + "reward_std": 0.6025157924741507, + "rewards/accuracy_reward": 0.3125000074505806, + "rewards/cosine_scaled_reward": 0.09110023453831673, + "step": 238 + }, + { + "clip_fraction": 0.0, + "completion_length": 1740.1666793823242, + "epoch": 0.27314285714285713, + "grad_norm": 0.09357193857431412, + "kl": 1.8790364265441895e-05, + "lambda_div_used": 0.5962883979082108, + "learning_rate": 6.649505910711058e-07, + "loss": 0.0055, + "reward": 0.07376761082559824, + "reward_after_mean": 0.07376761082559824, + "reward_after_std": 0.5891099888831377, + "reward_before_mean": 0.5742647312581539, + "reward_before_std": 0.44304153323173523, + "reward_change_max": 0.0, + "reward_change_mean": -0.5004971195012331, + "reward_change_min": -0.7436067499220371, + "reward_change_std": 0.2840570341795683, + "reward_std": 0.5891100075095892, + "rewards/accuracy_reward": 0.37500000186264515, + "rewards/cosine_scaled_reward": 0.19926471984945238, + "step": 239 + }, + { + "clip_fraction": 0.0, + "completion_length": 3087.416702270508, + "epoch": 0.2742857142857143, + "grad_norm": 0.06324295699596405, + "kl": 3.217160701751709e-05, + "lambda_div_used": 0.5339473709464073, + "learning_rate": 6.619104492241847e-07, + "loss": 0.0166, + "reward": -0.5358126908540726, + "reward_after_mean": -0.5358126908540726, + "reward_after_std": 0.23053276538848877, + "reward_before_mean": -0.2753155492246151, + "reward_before_std": 0.1490377252921462, + "reward_change_max": 0.0, + "reward_change_mean": -0.26049717888236046, + "reward_change_min": -0.3783254958689213, + "reward_change_std": 0.1383643699809909, + "reward_std": 0.23053277097642422, + "rewards/accuracy_reward": 0.0, + "rewards/cosine_scaled_reward": -0.2753155454993248, + "step": 240 + }, + { + "clip_fraction": 0.0, + "completion_length": 3419.625, + "epoch": 0.2754285714285714, + "grad_norm": 0.047396283596754074, + "kl": 3.555417060852051e-05, + "lambda_div_used": 0.5585425272583961, + "learning_rate": 6.588648530198504e-07, + "loss": 0.0027, + "reward": -0.4805446630343795, + "reward_after_mean": -0.4805446630343795, + "reward_after_std": 0.3355217073112726, + "reward_before_mean": -0.23604051489382982, + "reward_before_std": 0.2620125887915492, + "reward_change_max": 0.0, + "reward_change_mean": -0.2445041425526142, + "reward_change_min": -0.37724288925528526, + "reward_change_std": 0.13484715577214956, + "reward_std": 0.3355217222124338, + "rewards/accuracy_reward": 0.02083333395421505, + "rewards/cosine_scaled_reward": -0.2568738413974643, + "step": 241 + }, + { + "clip_fraction": 0.0, + "completion_length": 2400.8750228881836, + "epoch": 0.2765714285714286, + "grad_norm": 0.08974741399288177, + "kl": 4.5746564865112305e-05, + "lambda_div_used": 0.5887154564261436, + "learning_rate": 6.558139508961654e-07, + "loss": -0.0452, + "reward": -0.2555653927847743, + "reward_after_mean": -0.2555653927847743, + "reward_after_std": 0.47001883387565613, + "reward_before_mean": 0.03157716616988182, + "reward_before_std": 0.4061046461574733, + "reward_change_max": 0.0, + "reward_change_mean": -0.28714255429804325, + "reward_change_min": -0.41172971203923225, + "reward_change_std": 0.16218022629618645, + "reward_std": 0.47001886926591396, + "rewards/accuracy_reward": 0.1458333395421505, + "rewards/cosine_scaled_reward": -0.1142561559099704, + "step": 242 + }, + { + "clip_fraction": 0.0, + "completion_length": 2828.541679382324, + "epoch": 0.2777142857142857, + "grad_norm": 0.07105688005685806, + "kl": 1.8571503460407257e-05, + "lambda_div_used": 0.5722818151116371, + "learning_rate": 6.527578915497951e-07, + "loss": 0.0039, + "reward": -0.1431608721613884, + "reward_after_mean": -0.1431608721613884, + "reward_after_std": 0.42974008433520794, + "reward_before_mean": 0.2529455106705427, + "reward_before_std": 0.3276430475525558, + "reward_change_max": 0.0, + "reward_change_mean": -0.39610639959573746, + "reward_change_min": -0.5491434335708618, + "reward_change_std": 0.21527612209320068, + "reward_std": 0.42974009923636913, + "rewards/accuracy_reward": 0.25000000558793545, + "rewards/cosine_scaled_reward": 0.0029455198673531413, + "step": 243 + }, + { + "clip_fraction": 0.0, + "completion_length": 2882.291679382324, + "epoch": 0.27885714285714286, + "grad_norm": 0.0700126439332962, + "kl": 3.098323941230774e-05, + "lambda_div_used": 0.6161807402968407, + "learning_rate": 6.496968239287603e-07, + "loss": 0.0, + "reward": -0.06458889320492744, + "reward_after_mean": -0.06458889320492744, + "reward_after_std": 0.5576508566737175, + "reward_before_mean": 0.2605547234416008, + "reward_before_std": 0.5402053641155362, + "reward_change_max": 0.0, + "reward_change_mean": -0.32514358684420586, + "reward_change_min": -0.5207333639264107, + "reward_change_std": 0.21059911139309406, + "reward_std": 0.5576508603990078, + "rewards/accuracy_reward": 0.27083334140479565, + "rewards/cosine_scaled_reward": -0.01027863984927535, + "step": 244 + }, + { + "clip_fraction": 0.0, + "completion_length": 2907.4583587646484, + "epoch": 0.28, + "grad_norm": 0.05816182866692543, + "kl": 3.5434961318969727e-05, + "lambda_div_used": 0.5615689232945442, + "learning_rate": 6.466308972251785e-07, + "loss": -0.0133, + "reward": -0.42641448229551315, + "reward_after_mean": -0.42641448229551315, + "reward_after_std": 0.3361923936754465, + "reward_before_mean": -0.15409247018396854, + "reward_before_std": 0.2813150165602565, + "reward_change_max": 0.0, + "reward_change_mean": -0.2723220158368349, + "reward_change_min": -0.445806298404932, + "reward_change_std": 0.1635848032310605, + "reward_std": 0.3361923974007368, + "rewards/accuracy_reward": 0.0416666679084301, + "rewards/cosine_scaled_reward": -0.19575914042070508, + "step": 245 + }, + { + "clip_fraction": 0.0, + "completion_length": 2680.7083587646484, + "epoch": 0.28114285714285714, + "grad_norm": 0.07198330760002136, + "kl": 1.764507032930851e-05, + "lambda_div_used": 0.5865297466516495, + "learning_rate": 6.435602608679916e-07, + "loss": 0.0347, + "reward": -0.19568787328898907, + "reward_after_mean": -0.19568787328898907, + "reward_after_std": 0.4485197216272354, + "reward_before_mean": 0.129365224391222, + "reward_before_std": 0.393932550214231, + "reward_change_max": 0.0, + "reward_change_mean": -0.32505310885608196, + "reward_change_min": -0.4816659241914749, + "reward_change_std": 0.1897038472816348, + "reward_std": 0.44851974956691265, + "rewards/accuracy_reward": 0.20833334140479565, + "rewards/cosine_scaled_reward": -0.0789681114256382, + "step": 246 + }, + { + "clip_fraction": 0.0, + "completion_length": 3076.0208435058594, + "epoch": 0.2822857142857143, + "grad_norm": 0.08006418496370316, + "kl": 4.15705144405365e-05, + "lambda_div_used": 0.5735789015889168, + "learning_rate": 6.404850645156841e-07, + "loss": 0.0199, + "reward": -0.3997113136574626, + "reward_after_mean": -0.3997113136574626, + "reward_after_std": 0.41638931445777416, + "reward_before_mean": -0.1426885835826397, + "reward_before_std": 0.3330965582281351, + "reward_change_max": 0.0, + "reward_change_mean": -0.2570227347314358, + "reward_change_min": -0.38052143156528473, + "reward_change_std": 0.13911327440291643, + "reward_std": 0.4163893237709999, + "rewards/accuracy_reward": 0.0416666679084301, + "rewards/cosine_scaled_reward": -0.18435525218956172, + "step": 247 + }, + { + "clip_fraction": 0.0, + "completion_length": 2145.0000228881836, + "epoch": 0.2834285714285714, + "grad_norm": 0.10052043944597244, + "kl": 3.3289194107055664e-05, + "lambda_div_used": 0.6171368733048439, + "learning_rate": 6.374054580489873e-07, + "loss": 0.0514, + "reward": 0.28496517799794674, + "reward_after_mean": 0.28496517799794674, + "reward_after_std": 0.6654367055743933, + "reward_before_mean": 0.83577667362988, + "reward_before_std": 0.5420532608404756, + "reward_change_max": 0.0, + "reward_change_mean": -0.5508115068078041, + "reward_change_min": -0.7994196638464928, + "reward_change_std": 0.3166588256135583, + "reward_std": 0.6654367428272963, + "rewards/accuracy_reward": 0.5000000093132257, + "rewards/cosine_scaled_reward": 0.33577666338533163, + "step": 248 + }, + { + "clip_fraction": 0.0, + "completion_length": 2076.937515258789, + "epoch": 0.2845714285714286, + "grad_norm": 0.0908605083823204, + "kl": 2.4942681193351746e-05, + "lambda_div_used": 0.5948201194405556, + "learning_rate": 6.343215915635761e-07, + "loss": 0.0219, + "reward": -0.003894178196787834, + "reward_after_mean": -0.003894178196787834, + "reward_after_std": 0.5242901761084795, + "reward_before_mean": 0.4153097262606025, + "reward_before_std": 0.43534869560971856, + "reward_change_max": 0.0, + "reward_change_mean": -0.41920389235019684, + "reward_change_min": -0.5903440341353416, + "reward_change_std": 0.2394579891115427, + "reward_std": 0.524290194734931, + "rewards/accuracy_reward": 0.33333334140479565, + "rewards/cosine_scaled_reward": 0.08197638019919395, + "step": 249 + }, + { + "clip_fraction": 0.0, + "completion_length": 2605.0208587646484, + "epoch": 0.2857142857142857, + "grad_norm": 0.09027790278196335, + "kl": 3.804638981819153e-05, + "lambda_div_used": 0.5779460370540619, + "learning_rate": 6.31233615362752e-07, + "loss": -0.0589, + "reward": -0.2859988175332546, + "reward_after_mean": -0.2859988175332546, + "reward_after_std": 0.4080943390727043, + "reward_before_mean": 0.008792944252490997, + "reward_before_std": 0.35337654035538435, + "reward_change_max": 0.0, + "reward_change_mean": -0.2947917561978102, + "reward_change_min": -0.4365835040807724, + "reward_change_std": 0.16840537451207638, + "reward_std": 0.40809434466063976, + "rewards/accuracy_reward": 0.12500000558793545, + "rewards/cosine_scaled_reward": -0.11620705761015415, + "step": 250 + }, + { + "clip_fraction": 0.0, + "completion_length": 2368.9167137145996, + "epoch": 0.28685714285714287, + "grad_norm": 0.09640161693096161, + "kl": 2.7060508728027344e-05, + "lambda_div_used": 0.5987462773919106, + "learning_rate": 6.281416799501187e-07, + "loss": -0.0435, + "reward": -0.028017258271574974, + "reward_after_mean": -0.028017258271574974, + "reward_after_std": 0.5376799181103706, + "reward_before_mean": 0.38438196340575814, + "reward_before_std": 0.4528377316892147, + "reward_change_max": 0.0, + "reward_change_mean": -0.41239920631051064, + "reward_change_min": -0.6119380593299866, + "reward_change_std": 0.23853347077965736, + "reward_std": 0.5376799292862415, + "rewards/accuracy_reward": 0.33333334140479565, + "rewards/cosine_scaled_reward": 0.05104860384017229, + "step": 251 + }, + { + "clip_fraction": 0.0, + "completion_length": 2798.5625228881836, + "epoch": 0.288, + "grad_norm": 0.06784425675868988, + "kl": 3.535952419042587e-05, + "lambda_div_used": 0.5580189973115921, + "learning_rate": 6.25045936022246e-07, + "loss": 0.0335, + "reward": -0.2931139934808016, + "reward_after_mean": -0.2931139934808016, + "reward_after_std": 0.38731373474001884, + "reward_before_mean": 0.06481979880481958, + "reward_before_std": 0.2658566879108548, + "reward_change_max": 0.0, + "reward_change_mean": -0.3579337988048792, + "reward_change_min": -0.5358714908361435, + "reward_change_std": 0.19932966493070126, + "reward_std": 0.3873137477785349, + "rewards/accuracy_reward": 0.1666666679084301, + "rewards/cosine_scaled_reward": -0.10184688493609428, + "step": 252 + }, + { + "clip_fraction": 0.0, + "completion_length": 2946.416679382324, + "epoch": 0.28914285714285715, + "grad_norm": 0.09991417825222015, + "kl": 3.4242868423461914e-05, + "lambda_div_used": 0.6016674339771271, + "learning_rate": 6.219465344613258e-07, + "loss": -0.0465, + "reward": -0.0356330550275743, + "reward_after_mean": -0.0356330550275743, + "reward_after_std": 0.5561915785074234, + "reward_before_mean": 0.353804474696517, + "reward_before_std": 0.4667428769171238, + "reward_change_max": 0.0, + "reward_change_mean": -0.389437522739172, + "reward_change_min": -0.592089481651783, + "reward_change_std": 0.22914788126945496, + "reward_std": 0.5561915840953588, + "rewards/accuracy_reward": 0.31250000558793545, + "rewards/cosine_scaled_reward": 0.041304459096863866, + "step": 253 + }, + { + "clip_fraction": 0.0, + "completion_length": 2754.3541946411133, + "epoch": 0.29028571428571426, + "grad_norm": 0.08061102032661438, + "kl": 3.886967897415161e-05, + "lambda_div_used": 0.6251527816057205, + "learning_rate": 6.188436263278172e-07, + "loss": 0.0857, + "reward": -0.1664750911295414, + "reward_after_mean": -0.1664750911295414, + "reward_after_std": 0.599596256390214, + "reward_before_mean": 0.1026190984994173, + "reward_before_std": 0.5836509419605136, + "reward_change_max": 0.0, + "reward_change_mean": -0.2690941859036684, + "reward_change_min": -0.49960801005363464, + "reward_change_std": 0.19118957500904799, + "reward_std": 0.5995962955057621, + "rewards/accuracy_reward": 0.18750000186264515, + "rewards/cosine_scaled_reward": -0.08488089527236298, + "step": 254 + }, + { + "clip_fraction": 0.0, + "completion_length": 3190.083335876465, + "epoch": 0.2914285714285714, + "grad_norm": 0.08434556424617767, + "kl": 2.3322179913520813e-05, + "lambda_div_used": 0.5955284982919693, + "learning_rate": 6.157373628530852e-07, + "loss": -0.0125, + "reward": -0.3585695568472147, + "reward_after_mean": -0.3585695568472147, + "reward_after_std": 0.4996061436831951, + "reward_before_mean": -0.12086338270455599, + "reward_before_std": 0.4362535886466503, + "reward_change_max": 0.0, + "reward_change_mean": -0.2377061638981104, + "reward_change_min": -0.39885040931403637, + "reward_change_std": 0.1406589960679412, + "reward_std": 0.4996061585843563, + "rewards/accuracy_reward": 0.0833333358168602, + "rewards/cosine_scaled_reward": -0.20419671526178718, + "step": 255 + }, + { + "clip_fraction": 0.0, + "completion_length": 3160.8125610351562, + "epoch": 0.2925714285714286, + "grad_norm": 0.0576457642018795, + "kl": 3.143772482872009e-05, + "lambda_div_used": 0.639025017619133, + "learning_rate": 6.126278954320294e-07, + "loss": 0.0243, + "reward": -0.03833652473986149, + "reward_after_mean": -0.03833652473986149, + "reward_after_std": 0.6670235451310873, + "reward_before_mean": 0.2675688254312263, + "reward_before_std": 0.6473761759698391, + "reward_change_max": 0.0, + "reward_change_mean": -0.30590534023940563, + "reward_change_min": -0.511355496942997, + "reward_change_std": 0.2077637044712901, + "reward_std": 0.6670235693454742, + "rewards/accuracy_reward": 0.2500000037252903, + "rewards/cosine_scaled_reward": 0.01756881456822157, + "step": 256 + }, + { + "clip_fraction": 0.0, + "completion_length": 3138.5000228881836, + "epoch": 0.2937142857142857, + "grad_norm": 0.05595054104924202, + "kl": 3.0346214771270752e-05, + "lambda_div_used": 0.6474356725811958, + "learning_rate": 6.095153756157051e-07, + "loss": 0.025, + "reward": -0.0769930558744818, + "reward_after_mean": -0.0769930558744818, + "reward_after_std": 0.7138958293944597, + "reward_before_mean": 0.18525638710707426, + "reward_before_std": 0.685269920155406, + "reward_change_max": 0.0, + "reward_change_mean": -0.26224944926798344, + "reward_change_min": -0.4810149297118187, + "reward_change_std": 0.18088674824684858, + "reward_std": 0.7138958312571049, + "rewards/accuracy_reward": 0.2291666716337204, + "rewards/cosine_scaled_reward": -0.04391028080135584, + "step": 257 + }, + { + "clip_fraction": 0.0, + "completion_length": 3473.187530517578, + "epoch": 0.2948571428571429, + "grad_norm": 0.05202889442443848, + "kl": 3.561750054359436e-05, + "lambda_div_used": 0.6007983982563019, + "learning_rate": 6.06399955103937e-07, + "loss": 0.0058, + "reward": -0.23316247668117285, + "reward_after_mean": -0.23316247668117285, + "reward_after_std": 0.5119184870272875, + "reward_before_mean": 0.04610642418265343, + "reward_before_std": 0.46757086645811796, + "reward_change_max": 0.0, + "reward_change_mean": -0.2792689222842455, + "reward_change_min": -0.4602624364197254, + "reward_change_std": 0.17828952055424452, + "reward_std": 0.5119185000658035, + "rewards/accuracy_reward": 0.14583333395421505, + "rewards/cosine_scaled_reward": -0.09972689533606172, + "step": 258 + }, + { + "clip_fraction": 0.0, + "completion_length": 2996.7500534057617, + "epoch": 0.296, + "grad_norm": 0.11353152990341187, + "kl": 4.0959566831588745e-05, + "lambda_div_used": 0.605031318962574, + "learning_rate": 6.032817857379256e-07, + "loss": -0.0572, + "reward": -0.24131755530834198, + "reward_after_mean": -0.24131755530834198, + "reward_after_std": 0.5208989772945642, + "reward_before_mean": 0.03226012596860528, + "reward_before_std": 0.4831458665430546, + "reward_change_max": 0.0, + "reward_change_mean": -0.27357766777276993, + "reward_change_min": -0.4753991588950157, + "reward_change_std": 0.1725512887351215, + "reward_std": 0.5208989884704351, + "rewards/accuracy_reward": 0.14583333767950535, + "rewards/cosine_scaled_reward": -0.1135732214897871, + "step": 259 + }, + { + "clip_fraction": 0.0, + "completion_length": 2184.8542098999023, + "epoch": 0.29714285714285715, + "grad_norm": 0.1352783739566803, + "kl": 2.3871660232543945e-05, + "lambda_div_used": 0.6604775562882423, + "learning_rate": 6.001610194928464e-07, + "loss": 0.0496, + "reward": 0.1305767484009266, + "reward_after_mean": 0.1305767484009266, + "reward_after_std": 0.7162795849144459, + "reward_before_mean": 0.48802967881783843, + "reward_before_std": 0.7462537074461579, + "reward_change_max": 0.0, + "reward_change_mean": -0.3574528992176056, + "reward_change_min": -0.6257632970809937, + "reward_change_std": 0.25755990110337734, + "reward_std": 0.7162796072661877, + "rewards/accuracy_reward": 0.39583334513008595, + "rewards/cosine_scaled_reward": 0.0921963145956397, + "step": 260 + }, + { + "clip_fraction": 0.0, + "completion_length": 3070.3125228881836, + "epoch": 0.29828571428571427, + "grad_norm": 0.06917808949947357, + "kl": 3.3020973205566406e-05, + "lambda_div_used": 0.5845921337604523, + "learning_rate": 5.97037808470444e-07, + "loss": 0.0106, + "reward": -0.3141371374949813, + "reward_after_mean": -0.3141371374949813, + "reward_after_std": 0.43580205366015434, + "reward_before_mean": -0.03358305338770151, + "reward_before_std": 0.38949463702738285, + "reward_change_max": 0.0, + "reward_change_mean": -0.28055410645902157, + "reward_change_min": -0.44674795493483543, + "reward_change_std": 0.16879158467054367, + "reward_std": 0.4358020592480898, + "rewards/accuracy_reward": 0.08333333395421505, + "rewards/cosine_scaled_reward": -0.11691637896001339, + "step": 261 + }, + { + "clip_fraction": 0.0, + "completion_length": 3258.3125, + "epoch": 0.29942857142857143, + "grad_norm": 0.06239396706223488, + "kl": 4.535168409347534e-05, + "lambda_div_used": 0.5396690741181374, + "learning_rate": 5.939123048916173e-07, + "loss": 0.0153, + "reward": -0.49852147325873375, + "reward_after_mean": -0.49852147325873375, + "reward_after_std": 0.23993146419525146, + "reward_before_mean": -0.21930878423154354, + "reward_before_std": 0.17422470543533564, + "reward_change_max": 0.0, + "reward_change_mean": -0.2792126890271902, + "reward_change_min": -0.4082174263894558, + "reward_change_std": 0.15261581167578697, + "reward_std": 0.2399314697831869, + "rewards/accuracy_reward": 0.0, + "rewards/cosine_scaled_reward": -0.21930878423154354, + "step": 262 + }, + { + "clip_fraction": 0.0, + "completion_length": 2837.6666717529297, + "epoch": 0.30057142857142854, + "grad_norm": 0.07131204754114151, + "kl": 2.771243453025818e-05, + "lambda_div_used": 0.5753477811813354, + "learning_rate": 5.907846610890011e-07, + "loss": 0.0159, + "reward": -0.32574891671538353, + "reward_after_mean": -0.32574891671538353, + "reward_after_std": 0.40547293052077293, + "reward_before_mean": -0.04349888768047094, + "reward_before_std": 0.3437284992542118, + "reward_change_max": 0.0, + "reward_change_mean": -0.28225001133978367, + "reward_change_min": -0.423517182469368, + "reward_change_std": 0.16225541010499, + "reward_std": 0.4054729398339987, + "rewards/accuracy_reward": 0.10416666977107525, + "rewards/cosine_scaled_reward": -0.14766556210815907, + "step": 263 + }, + { + "clip_fraction": 0.0, + "completion_length": 2913.812515258789, + "epoch": 0.3017142857142857, + "grad_norm": 0.07276218384504318, + "kl": 3.5703182220458984e-05, + "lambda_div_used": 0.5857644975185394, + "learning_rate": 5.87655029499542e-07, + "loss": 0.039, + "reward": -0.19073306024074554, + "reward_after_mean": -0.19073306024074554, + "reward_after_std": 0.4183583725243807, + "reward_before_mean": 0.13579276762902737, + "reward_before_std": 0.39237749949097633, + "reward_change_max": 0.0, + "reward_change_mean": -0.32652581483125687, + "reward_change_min": -0.512863963842392, + "reward_change_std": 0.20149299688637257, + "reward_std": 0.4183583725243807, + "rewards/accuracy_reward": 0.1875000074505806, + "rewards/cosine_scaled_reward": -0.051707250997424126, + "step": 264 + }, + { + "clip_fraction": 0.0, + "completion_length": 2092.333351135254, + "epoch": 0.3028571428571429, + "grad_norm": 0.09631343185901642, + "kl": 2.018176019191742e-05, + "lambda_div_used": 0.5927653685212135, + "learning_rate": 5.845235626570683e-07, + "loss": 0.0409, + "reward": -0.05978839658200741, + "reward_after_mean": -0.05978839658200741, + "reward_after_std": 0.5236263573169708, + "reward_before_mean": 0.3295632619410753, + "reward_before_std": 0.4253385625779629, + "reward_change_max": 0.0, + "reward_change_mean": -0.38935166224837303, + "reward_change_min": -0.5855174511671066, + "reward_change_std": 0.22452097665518522, + "reward_std": 0.5236263833940029, + "rewards/accuracy_reward": 0.29166666977107525, + "rewards/cosine_scaled_reward": 0.037896597757935524, + "step": 265 + }, + { + "clip_fraction": 0.0, + "completion_length": 3210.8541717529297, + "epoch": 0.304, + "grad_norm": 0.05451129376888275, + "kl": 3.138929605484009e-05, + "lambda_div_used": 0.557657316327095, + "learning_rate": 5.813904131848564e-07, + "loss": -0.0024, + "reward": -0.3950183019042015, + "reward_after_mean": -0.3950183019042015, + "reward_after_std": 0.32154187746345997, + "reward_before_mean": -0.12019951082766056, + "reward_before_std": 0.2612606221809983, + "reward_change_max": 0.0, + "reward_change_mean": -0.2748187892138958, + "reward_change_min": -0.42080841958522797, + "reward_change_std": 0.158494733273983, + "reward_std": 0.321541890501976, + "rewards/accuracy_reward": 0.0833333358168602, + "rewards/cosine_scaled_reward": -0.20353284664452076, + "step": 266 + }, + { + "clip_fraction": 0.0, + "completion_length": 3039.916679382324, + "epoch": 0.30514285714285716, + "grad_norm": 0.06252676248550415, + "kl": 3.5075165214948356e-05, + "lambda_div_used": 0.5528770685195923, + "learning_rate": 5.78255733788191e-07, + "loss": -0.037, + "reward": -0.4460434205830097, + "reward_after_mean": -0.4460434205830097, + "reward_after_std": 0.3107005339115858, + "reward_before_mean": -0.1744655454531312, + "reward_before_std": 0.2377403611317277, + "reward_change_max": 0.0, + "reward_change_mean": -0.27157786674797535, + "reward_change_min": -0.42647456377744675, + "reward_change_std": 0.1541104121133685, + "reward_std": 0.3107005413621664, + "rewards/accuracy_reward": 0.0416666679084301, + "rewards/cosine_scaled_reward": -0.21613221243023872, + "step": 267 + }, + { + "clip_fraction": 0.0, + "completion_length": 2704.8333587646484, + "epoch": 0.3062857142857143, + "grad_norm": 0.08417963981628418, + "kl": 4.7713518142700195e-05, + "lambda_div_used": 0.6233388632535934, + "learning_rate": 5.751196772469237e-07, + "loss": 0.0822, + "reward": -0.21941478177905083, + "reward_after_mean": -0.21941478177905083, + "reward_after_std": 0.6098091676831245, + "reward_before_mean": 0.03345827816519886, + "reward_before_std": 0.5743174999952316, + "reward_change_max": 0.0, + "reward_change_mean": -0.25287305377423763, + "reward_change_min": -0.45724478363990784, + "reward_change_std": 0.1690685572102666, + "reward_std": 0.6098091900348663, + "rewards/accuracy_reward": 0.1458333358168602, + "rewards/cosine_scaled_reward": -0.11237506754696369, + "step": 268 + }, + { + "clip_fraction": 0.0, + "completion_length": 3165.9791717529297, + "epoch": 0.30742857142857144, + "grad_norm": 0.06371315568685532, + "kl": 2.181902527809143e-05, + "lambda_div_used": 0.5798910111188889, + "learning_rate": 5.71982396408026e-07, + "loss": 0.0477, + "reward": -0.38025568798184395, + "reward_after_mean": -0.38025568798184395, + "reward_after_std": 0.4126162938773632, + "reward_before_mean": -0.13023450784385204, + "reward_before_std": 0.3629760518670082, + "reward_change_max": 0.0, + "reward_change_mean": -0.2500211838632822, + "reward_change_min": -0.40024495497345924, + "reward_change_std": 0.15232266392558813, + "reward_std": 0.4126163087785244, + "rewards/accuracy_reward": 0.08333333395421505, + "rewards/cosine_scaled_reward": -0.2135678417980671, + "step": 269 + }, + { + "clip_fraction": 0.0, + "completion_length": 3056.875045776367, + "epoch": 0.30857142857142855, + "grad_norm": 0.05920446664094925, + "kl": 2.387911081314087e-05, + "lambda_div_used": 0.6503699943423271, + "learning_rate": 5.688440441781398e-07, + "loss": 0.0, + "reward": -0.004786740057170391, + "reward_after_mean": -0.004786740057170391, + "reward_after_std": 0.6943319551646709, + "reward_before_mean": 0.292231020051986, + "reward_before_std": 0.7036092299968004, + "reward_change_max": 0.0, + "reward_change_mean": -0.2970177587121725, + "reward_change_min": -0.5634518079459667, + "reward_change_std": 0.21872046310454607, + "reward_std": 0.6943319924175739, + "rewards/accuracy_reward": 0.27083333767950535, + "rewards/cosine_scaled_reward": 0.021397670439910144, + "step": 270 + }, + { + "clip_fraction": 0.0, + "completion_length": 2221.3958702087402, + "epoch": 0.3097142857142857, + "grad_norm": 0.09025074541568756, + "kl": 6.723217666149139e-06, + "lambda_div_used": 0.640851192176342, + "learning_rate": 5.657047735161255e-07, + "loss": 0.0064, + "reward": 0.11755780689418316, + "reward_after_mean": 0.11755780689418316, + "reward_after_std": 0.6356876939535141, + "reward_before_mean": 0.4748641401529312, + "reward_before_std": 0.654548792168498, + "reward_change_max": 0.0, + "reward_change_mean": -0.35730634443461895, + "reward_change_min": -0.5644437614828348, + "reward_change_std": 0.23989163804799318, + "reward_std": 0.6356877088546753, + "rewards/accuracy_reward": 0.39583334885537624, + "rewards/cosine_scaled_reward": 0.07903079688549042, + "step": 271 + }, + { + "clip_fraction": 0.0, + "completion_length": 2814.5208740234375, + "epoch": 0.31085714285714283, + "grad_norm": 0.07214022427797318, + "kl": 2.553686499595642e-05, + "lambda_div_used": 0.5532237812876701, + "learning_rate": 5.625647374256061e-07, + "loss": -0.023, + "reward": -0.33643968403339386, + "reward_after_mean": -0.33643968403339386, + "reward_after_std": 0.35168860107660294, + "reward_before_mean": 0.01186647079885006, + "reward_before_std": 0.2353415172547102, + "reward_change_max": 0.0, + "reward_change_mean": -0.34830615669488907, + "reward_change_min": -0.5079392194747925, + "reward_change_std": 0.1846030419692397, + "reward_std": 0.351688614115119, + "rewards/accuracy_reward": 0.14583333395421505, + "rewards/cosine_scaled_reward": -0.13396686874330044, + "step": 272 + }, + { + "clip_fraction": 0.0, + "completion_length": 2673.937515258789, + "epoch": 0.312, + "grad_norm": 0.067450150847435, + "kl": 2.347538247704506e-05, + "lambda_div_used": 0.5976931154727936, + "learning_rate": 5.594240889475106e-07, + "loss": 0.0439, + "reward": 0.060379184782505035, + "reward_after_mean": 0.060379184782505035, + "reward_after_std": 0.5129444599151611, + "reward_before_mean": 0.4946505483239889, + "reward_before_std": 0.44454328902065754, + "reward_change_max": 0.0, + "reward_change_mean": -0.4342713989317417, + "reward_change_min": -0.6668536812067032, + "reward_change_std": 0.25412870943546295, + "reward_std": 0.5129444822669029, + "rewards/accuracy_reward": 0.3750000111758709, + "rewards/cosine_scaled_reward": 0.1196505706757307, + "step": 273 + }, + { + "clip_fraction": 0.0, + "completion_length": 1866.4375076293945, + "epoch": 0.31314285714285717, + "grad_norm": 0.11823767423629761, + "kl": 2.6108697056770325e-05, + "lambda_div_used": 0.6173145174980164, + "learning_rate": 5.562829811526154e-07, + "loss": -0.0666, + "reward": 0.20245935022830963, + "reward_after_mean": 0.20245935022830963, + "reward_after_std": 0.6078098546713591, + "reward_before_mean": 0.6919816123554483, + "reward_before_std": 0.5468513960950077, + "reward_change_max": 0.0, + "reward_change_mean": -0.4895222559571266, + "reward_change_min": -0.7379350513219833, + "reward_change_std": 0.3024911228567362, + "reward_std": 0.6078098695725203, + "rewards/accuracy_reward": 0.4583333432674408, + "rewards/cosine_scaled_reward": 0.2336482839891687, + "step": 274 + }, + { + "clip_fraction": 0.0, + "completion_length": 2334.0416870117188, + "epoch": 0.3142857142857143, + "grad_norm": 0.07390157133340836, + "kl": 2.975761890411377e-05, + "lambda_div_used": 0.5953021869063377, + "learning_rate": 5.531415671340826e-07, + "loss": 0.0067, + "reward": -0.02193348854780197, + "reward_after_mean": -0.02193348854780197, + "reward_after_std": 0.5273136273026466, + "reward_before_mean": 0.3926870714276447, + "reward_before_std": 0.4364564120769501, + "reward_change_max": 0.0, + "reward_change_mean": -0.4146205447614193, + "reward_change_min": -0.6028474234044552, + "reward_change_std": 0.23770872876048088, + "reward_std": 0.5273136328905821, + "rewards/accuracy_reward": 0.31250000558793545, + "rewards/cosine_scaled_reward": 0.08018706925213337, + "step": 275 + }, + { + "clip_fraction": 0.0, + "completion_length": 2735.6458740234375, + "epoch": 0.31542857142857145, + "grad_norm": 0.07891591638326645, + "kl": 3.522634506225586e-05, + "lambda_div_used": 0.6019129753112793, + "learning_rate": 5.5e-07, + "loss": -0.0433, + "reward": 0.016597013920545578, + "reward_after_mean": 0.016597013920545578, + "reward_after_std": 0.5496951006352901, + "reward_before_mean": 0.43198024667799473, + "reward_before_std": 0.4716393407434225, + "reward_change_max": 0.0, + "reward_change_mean": -0.4153832569718361, + "reward_change_min": -0.6486029289662838, + "reward_change_std": 0.25036295782774687, + "reward_std": 0.5496951248496771, + "rewards/accuracy_reward": 0.3333333395421505, + "rewards/cosine_scaled_reward": 0.09864690899848938, + "step": 276 + }, + { + "clip_fraction": 0.0, + "completion_length": 2435.437530517578, + "epoch": 0.31657142857142856, + "grad_norm": 0.08397295325994492, + "kl": 2.7611851692199707e-05, + "lambda_div_used": 0.6150463595986366, + "learning_rate": 5.468584328659172e-07, + "loss": 0.0536, + "reward": -0.049143560230731964, + "reward_after_mean": -0.049143560230731964, + "reward_after_std": 0.5724204778671265, + "reward_before_mean": 0.29686027206480503, + "reward_before_std": 0.5315626971423626, + "reward_change_max": 0.0, + "reward_change_mean": -0.34600381925702095, + "reward_change_min": -0.5762605480849743, + "reward_change_std": 0.21639508474618196, + "reward_std": 0.5724204815924168, + "rewards/accuracy_reward": 0.27083333767950535, + "rewards/cosine_scaled_reward": 0.02602693811058998, + "step": 277 + }, + { + "clip_fraction": 0.0, + "completion_length": 2029.708381652832, + "epoch": 0.3177142857142857, + "grad_norm": 0.11247767508029938, + "kl": 3.094971179962158e-05, + "lambda_div_used": 0.6170216798782349, + "learning_rate": 5.437170188473847e-07, + "loss": 0.002, + "reward": 0.056462954729795456, + "reward_after_mean": 0.056462954729795456, + "reward_after_std": 0.6216260213404894, + "reward_before_mean": 0.48691817931830883, + "reward_before_std": 0.5461275167763233, + "reward_change_max": 0.0, + "reward_change_mean": -0.4304552264511585, + "reward_change_min": -0.7186719551682472, + "reward_change_std": 0.27375681325793266, + "reward_std": 0.6216260306537151, + "rewards/accuracy_reward": 0.3541666716337204, + "rewards/cosine_scaled_reward": 0.13275149278342724, + "step": 278 + }, + { + "clip_fraction": 0.0, + "completion_length": 3158.6875228881836, + "epoch": 0.31885714285714284, + "grad_norm": 0.06945524364709854, + "kl": 3.522634506225586e-05, + "lambda_div_used": 0.5564287528395653, + "learning_rate": 5.405759110524894e-07, + "loss": -0.0272, + "reward": -0.43442236818373203, + "reward_after_mean": -0.43442236818373203, + "reward_after_std": 0.32187592424452305, + "reward_before_mean": -0.1544840056449175, + "reward_before_std": 0.25075172632932663, + "reward_change_max": 0.0, + "reward_change_mean": -0.27993838116526604, + "reward_change_min": -0.41151439025998116, + "reward_change_std": 0.15283891931176186, + "reward_std": 0.32187593914568424, + "rewards/accuracy_reward": 0.02083333395421505, + "rewards/cosine_scaled_reward": -0.17531732900533825, + "step": 279 + }, + { + "clip_fraction": 0.0, + "completion_length": 2157.833335876465, + "epoch": 0.32, + "grad_norm": 0.14424921572208405, + "kl": 4.202499985694885e-05, + "lambda_div_used": 0.630903884768486, + "learning_rate": 5.37435262574394e-07, + "loss": -0.0339, + "reward": -0.14438428170979023, + "reward_after_mean": -0.14438428170979023, + "reward_after_std": 0.6414528246968985, + "reward_before_mean": 0.1313652544049546, + "reward_before_std": 0.6011563409119844, + "reward_change_max": 0.0, + "reward_change_mean": -0.275749534368515, + "reward_change_min": -0.4570390097796917, + "reward_change_std": 0.1709576854482293, + "reward_std": 0.6414528302848339, + "rewards/accuracy_reward": 0.20833334140479565, + "rewards/cosine_scaled_reward": -0.076968085631961, + "step": 280 + }, + { + "clip_fraction": 0.0, + "completion_length": 3559.000030517578, + "epoch": 0.3211428571428571, + "grad_norm": 0.0494488850235939, + "kl": 3.0197203159332275e-05, + "lambda_div_used": 0.5301951244473457, + "learning_rate": 5.342952264838747e-07, + "loss": -0.0035, + "reward": -0.5142169296741486, + "reward_after_mean": -0.5142169296741486, + "reward_after_std": 0.22926145792007446, + "reward_before_mean": -0.2300790660083294, + "reward_before_std": 0.1324998252093792, + "reward_change_max": 0.0, + "reward_change_mean": -0.2841378580778837, + "reward_change_min": -0.4100039228796959, + "reward_change_std": 0.14742697216570377, + "reward_std": 0.22926146537065506, + "rewards/accuracy_reward": 0.0, + "rewards/cosine_scaled_reward": -0.2300790660083294, + "step": 281 + }, + { + "clip_fraction": 0.0, + "completion_length": 2757.7708587646484, + "epoch": 0.3222857142857143, + "grad_norm": 0.07136296480894089, + "kl": 1.5236437320709229e-05, + "lambda_div_used": 0.6041858941316605, + "learning_rate": 5.311559558218603e-07, + "loss": 0.0304, + "reward": 0.016035709530115128, + "reward_after_mean": 0.016035709530115128, + "reward_after_std": 0.5573885068297386, + "reward_before_mean": 0.4244079850614071, + "reward_before_std": 0.48719789227470756, + "reward_change_max": 0.0, + "reward_change_mean": -0.40837226063013077, + "reward_change_min": -0.630510251969099, + "reward_change_std": 0.253003865480423, + "reward_std": 0.5573885291814804, + "rewards/accuracy_reward": 0.3541666716337204, + "rewards/cosine_scaled_reward": 0.07024132460355759, + "step": 282 + }, + { + "clip_fraction": 0.0, + "completion_length": 2882.5416946411133, + "epoch": 0.32342857142857145, + "grad_norm": 0.05982055515050888, + "kl": 1.1865049600601196e-05, + "lambda_div_used": 0.6191478446125984, + "learning_rate": 5.28017603591974e-07, + "loss": -0.015, + "reward": 0.10000502690672874, + "reward_after_mean": 0.10000502690672874, + "reward_after_std": 0.6287181153893471, + "reward_before_mean": 0.5279214177280664, + "reward_before_std": 0.5531216450035572, + "reward_change_max": 0.0, + "reward_change_mean": -0.4279164057224989, + "reward_change_min": -0.6583010666072369, + "reward_change_std": 0.2602456407621503, + "reward_std": 0.6287181228399277, + "rewards/accuracy_reward": 0.37500000931322575, + "rewards/cosine_scaled_reward": 0.1529214084148407, + "step": 283 + }, + { + "clip_fraction": 0.0, + "completion_length": 2187.083381652832, + "epoch": 0.32457142857142857, + "grad_norm": 0.20831985771656036, + "kl": 5.378853529691696e-05, + "lambda_div_used": 0.617803268134594, + "learning_rate": 5.248803227530763e-07, + "loss": 0.0301, + "reward": -0.09672576747834682, + "reward_after_mean": -0.09672576747834682, + "reward_after_std": 0.5820088647305965, + "reward_before_mean": 0.21344758570194244, + "reward_before_std": 0.544846129603684, + "reward_change_max": 0.0, + "reward_change_mean": -0.3101733736693859, + "reward_change_min": -0.5270103476941586, + "reward_change_std": 0.20029443874955177, + "reward_std": 0.5820088759064674, + "rewards/accuracy_reward": 0.22916667349636555, + "rewards/cosine_scaled_reward": -0.0157190952450037, + "step": 284 + }, + { + "clip_fraction": 0.0, + "completion_length": 2571.4583435058594, + "epoch": 0.32571428571428573, + "grad_norm": 0.05750874802470207, + "kl": 2.5823712348937988e-05, + "lambda_div_used": 0.5412982106208801, + "learning_rate": 5.21744266211809e-07, + "loss": 0.0429, + "reward": -0.3610886335372925, + "reward_after_mean": -0.3610886335372925, + "reward_after_std": 0.2963402010500431, + "reward_before_mean": 0.004524916410446167, + "reward_before_std": 0.18141429405659437, + "reward_change_max": 0.0, + "reward_change_mean": -0.36561354249715805, + "reward_change_min": -0.5212092585861683, + "reward_change_std": 0.19485185854136944, + "reward_std": 0.29634021408855915, + "rewards/accuracy_reward": 0.125, + "rewards/cosine_scaled_reward": -0.12047509290277958, + "step": 285 + }, + { + "clip_fraction": 0.0, + "completion_length": 2607.625030517578, + "epoch": 0.32685714285714285, + "grad_norm": 0.06422320753335953, + "kl": 2.036895602941513e-05, + "lambda_div_used": 0.593349277973175, + "learning_rate": 5.186095868151436e-07, + "loss": 0.0069, + "reward": -0.1434864792972803, + "reward_after_mean": -0.1434864792972803, + "reward_after_std": 0.4716298431158066, + "reward_before_mean": 0.18680993653833866, + "reward_before_std": 0.4280705275014043, + "reward_change_max": 0.0, + "reward_change_mean": -0.3302964009344578, + "reward_change_min": -0.5444469898939133, + "reward_change_std": 0.20489494875073433, + "reward_std": 0.4716298636049032, + "rewards/accuracy_reward": 0.2291666753590107, + "rewards/cosine_scaled_reward": -0.04235674999654293, + "step": 286 + }, + { + "clip_fraction": 0.0, + "completion_length": 2068.4375343322754, + "epoch": 0.328, + "grad_norm": 0.11217836290597916, + "kl": 3.282725811004639e-05, + "lambda_div_used": 0.576636016368866, + "learning_rate": 5.154764373429315e-07, + "loss": -0.0127, + "reward": -0.015125550329685211, + "reward_after_mean": -0.015125550329685211, + "reward_after_std": 0.44022079929709435, + "reward_before_mean": 0.4338516741991043, + "reward_before_std": 0.349846001714468, + "reward_change_max": 0.0, + "reward_change_mean": -0.4489772208034992, + "reward_change_min": -0.6400703266263008, + "reward_change_std": 0.25500839948654175, + "reward_std": 0.44022080302238464, + "rewards/accuracy_reward": 0.3125000074505806, + "rewards/cosine_scaled_reward": 0.12135165929794312, + "step": 287 + }, + { + "clip_fraction": 0.0, + "completion_length": 3021.1666870117188, + "epoch": 0.3291428571428571, + "grad_norm": 0.05326732248067856, + "kl": 2.1582163753919303e-05, + "lambda_div_used": 0.5814583152532578, + "learning_rate": 5.123449705004581e-07, + "loss": 0.0215, + "reward": -0.23482287488877773, + "reward_after_mean": -0.23482287488877773, + "reward_after_std": 0.4797380156815052, + "reward_before_mean": 0.10618079453706741, + "reward_before_std": 0.37177785113453865, + "reward_change_max": 0.0, + "reward_change_mean": -0.34100368432700634, + "reward_change_min": -0.5246531553566456, + "reward_change_std": 0.190623770467937, + "reward_std": 0.47973802499473095, + "rewards/accuracy_reward": 0.18750000186264515, + "rewards/cosine_scaled_reward": -0.08131920825690031, + "step": 288 + }, + { + "clip_fraction": 0.0, + "completion_length": 2210.0208435058594, + "epoch": 0.3302857142857143, + "grad_norm": 0.0918356254696846, + "kl": 3.3445656299591064e-05, + "lambda_div_used": 0.5579611882567406, + "learning_rate": 5.09215338910999e-07, + "loss": 0.0308, + "reward": -0.1890019178390503, + "reward_after_mean": -0.1890019178390503, + "reward_after_std": 0.38851393200457096, + "reward_before_mean": 0.2269407268613577, + "reward_before_std": 0.2580757178366184, + "reward_change_max": 0.0, + "reward_change_mean": -0.4159426633268595, + "reward_change_min": -0.5701811872422695, + "reward_change_std": 0.21641669981181622, + "reward_std": 0.38851393945515156, + "rewards/accuracy_reward": 0.27083333395421505, + "rewards/cosine_scaled_reward": -0.04389259871095419, + "step": 289 + }, + { + "clip_fraction": 0.0, + "completion_length": 1820.6666870117188, + "epoch": 0.3314285714285714, + "grad_norm": 0.10334110260009766, + "kl": 1.7639249563217163e-05, + "lambda_div_used": 0.649879202246666, + "learning_rate": 5.060876951083828e-07, + "loss": -0.0389, + "reward": 0.04819735325872898, + "reward_after_mean": 0.04819735325872898, + "reward_after_std": 0.752808591350913, + "reward_before_mean": 0.39133079699240625, + "reward_before_std": 0.7000208692625165, + "reward_change_max": 0.0, + "reward_change_mean": -0.3431334514170885, + "reward_change_min": -0.5988858677446842, + "reward_change_std": 0.2255254928022623, + "reward_std": 0.7528086155653, + "rewards/accuracy_reward": 0.3333333358168602, + "rewards/cosine_scaled_reward": 0.05799746699631214, + "step": 290 + }, + { + "clip_fraction": 0.0, + "completion_length": 2717.166732788086, + "epoch": 0.3325714285714286, + "grad_norm": 0.07149659842252731, + "kl": 9.842216968536377e-06, + "lambda_div_used": 0.6230586618185043, + "learning_rate": 5.02962191529556e-07, + "loss": 0.0634, + "reward": 0.04544975981116295, + "reward_after_mean": 0.04544975981116295, + "reward_after_std": 0.6337935384362936, + "reward_before_mean": 0.4299341347068548, + "reward_before_std": 0.5707205794751644, + "reward_change_max": 0.0, + "reward_change_mean": -0.38448438234627247, + "reward_change_min": -0.6367709413170815, + "reward_change_std": 0.2398422248661518, + "reward_std": 0.6337935607880354, + "rewards/accuracy_reward": 0.33333333767950535, + "rewards/cosine_scaled_reward": 0.09660080214962363, + "step": 291 + }, + { + "clip_fraction": 0.0, + "completion_length": 3176.979202270508, + "epoch": 0.33371428571428574, + "grad_norm": 0.05150657147169113, + "kl": 2.0240433514118195e-05, + "lambda_div_used": 0.5560602247714996, + "learning_rate": 4.998389805071536e-07, + "loss": 0.062, + "reward": -0.4294139966368675, + "reward_after_mean": -0.4294139966368675, + "reward_after_std": 0.3452510107308626, + "reward_before_mean": -0.15922172274440527, + "reward_before_std": 0.2578093442134559, + "reward_change_max": 0.0, + "reward_change_mean": -0.27019229158759117, + "reward_change_min": -0.42562897875905037, + "reward_change_std": 0.15438843425363302, + "reward_std": 0.3452510181814432, + "rewards/accuracy_reward": 0.0416666679084301, + "rewards/cosine_scaled_reward": -0.20088838692754507, + "step": 292 + }, + { + "clip_fraction": 0.0, + "completion_length": 2630.9791946411133, + "epoch": 0.33485714285714285, + "grad_norm": 0.05954331159591675, + "kl": 1.2833625078201294e-05, + "lambda_div_used": 0.5555919855833054, + "learning_rate": 4.967182142620745e-07, + "loss": 0.0009, + "reward": -0.19852645695209503, + "reward_after_mean": -0.19852645695209503, + "reward_after_std": 0.3515181578695774, + "reward_before_mean": 0.20652466267347336, + "reward_before_std": 0.24862384609878063, + "reward_change_max": 0.0, + "reward_change_mean": -0.40505112148821354, + "reward_change_min": -0.5747470110654831, + "reward_change_std": 0.22049889154732227, + "reward_std": 0.35151816345751286, + "rewards/accuracy_reward": 0.2291666716337204, + "rewards/cosine_scaled_reward": -0.022642023861408234, + "step": 293 + }, + { + "clip_fraction": 0.0, + "completion_length": 3005.5, + "epoch": 0.336, + "grad_norm": 0.06993379443883896, + "kl": 2.8274953365325928e-05, + "lambda_div_used": 0.5608418732881546, + "learning_rate": 4.93600044896063e-07, + "loss": -0.0248, + "reward": -0.3667301833629608, + "reward_after_mean": -0.3667301833629608, + "reward_after_std": 0.32350156269967556, + "reward_before_mean": -0.07280110754072666, + "reward_before_std": 0.2737845163792372, + "reward_change_max": 0.0, + "reward_change_mean": -0.29392906464636326, + "reward_change_min": -0.4593726359307766, + "reward_change_std": 0.17168805841356516, + "reward_std": 0.3235015720129013, + "rewards/accuracy_reward": 0.0833333358168602, + "rewards/cosine_scaled_reward": -0.15613443590700626, + "step": 294 + }, + { + "clip_fraction": 0.0, + "completion_length": 3221.2291717529297, + "epoch": 0.33714285714285713, + "grad_norm": 0.06364311277866364, + "kl": 1.817569136619568e-05, + "lambda_div_used": 0.5979067236185074, + "learning_rate": 4.904846243842949e-07, + "loss": 0.0107, + "reward": -0.259306401014328, + "reward_after_mean": -0.259306401014328, + "reward_after_std": 0.5024331342428923, + "reward_before_mean": 0.018260781886056066, + "reward_before_std": 0.45087322127074003, + "reward_change_max": 0.0, + "reward_change_mean": -0.277567183598876, + "reward_change_min": -0.41658853366971016, + "reward_change_std": 0.1643626783043146, + "reward_std": 0.5024331398308277, + "rewards/accuracy_reward": 0.1250000037252903, + "rewards/cosine_scaled_reward": -0.10673921927809715, + "step": 295 + }, + { + "clip_fraction": 0.0, + "completion_length": 3282.8750610351562, + "epoch": 0.3382857142857143, + "grad_norm": 0.0698896273970604, + "kl": 2.3877248167991638e-05, + "lambda_div_used": 0.5625706240534782, + "learning_rate": 4.873721045679706e-07, + "loss": 0.0079, + "reward": -0.39721857756376266, + "reward_after_mean": -0.39721857756376266, + "reward_after_std": 0.34284412302076817, + "reward_before_mean": -0.1091517936438322, + "reward_before_std": 0.28544116113334894, + "reward_change_max": 0.0, + "reward_change_mean": -0.28806679882109165, + "reward_change_min": -0.4639837518334389, + "reward_change_std": 0.1706175785511732, + "reward_std": 0.3428441286087036, + "rewards/accuracy_reward": 0.0416666679084301, + "rewards/cosine_scaled_reward": -0.15081845596432686, + "step": 296 + }, + { + "clip_fraction": 0.0, + "completion_length": 3569.8541870117188, + "epoch": 0.3394285714285714, + "grad_norm": 0.04541197791695595, + "kl": 2.245139330625534e-05, + "lambda_div_used": 0.5649159774184227, + "learning_rate": 4.842626371469149e-07, + "loss": 0.0071, + "reward": -0.43140796944499016, + "reward_after_mean": -0.43140796944499016, + "reward_after_std": 0.3433863054960966, + "reward_before_mean": -0.17089181207120419, + "reward_before_std": 0.29400468710809946, + "reward_change_max": 0.0, + "reward_change_mean": -0.26051616482436657, + "reward_change_min": -0.4413977116346359, + "reward_change_std": 0.15714262332767248, + "reward_std": 0.3433863129466772, + "rewards/accuracy_reward": 0.0416666679084301, + "rewards/cosine_scaled_reward": -0.2125584715977311, + "step": 297 + }, + { + "clip_fraction": 0.0, + "completion_length": 2834.479202270508, + "epoch": 0.3405714285714286, + "grad_norm": 0.07559877634048462, + "kl": 1.9371509552001953e-05, + "lambda_div_used": 0.5950812250375748, + "learning_rate": 4.811563736721829e-07, + "loss": -0.0217, + "reward": -0.13096300419420004, + "reward_after_mean": -0.13096300419420004, + "reward_after_std": 0.505608232691884, + "reward_before_mean": 0.22305661533027887, + "reward_before_std": 0.43375879526138306, + "reward_change_max": 0.0, + "reward_change_mean": -0.354019645601511, + "reward_change_min": -0.5659407489001751, + "reward_change_std": 0.20670694950968027, + "reward_std": 0.5056082457304001, + "rewards/accuracy_reward": 0.29166667349636555, + "rewards/cosine_scaled_reward": -0.0686100497841835, + "step": 298 + }, + { + "clip_fraction": 0.0, + "completion_length": 3288.5833587646484, + "epoch": 0.3417142857142857, + "grad_norm": 0.05182049795985222, + "kl": 3.013480454683304e-05, + "lambda_div_used": 0.5971293970942497, + "learning_rate": 4.780534655386743e-07, + "loss": -0.0035, + "reward": -0.1649590004235506, + "reward_after_mean": -0.1649590004235506, + "reward_after_std": 0.476910138502717, + "reward_before_mean": 0.15552489459514618, + "reward_before_std": 0.44229450821876526, + "reward_change_max": 0.0, + "reward_change_mean": -0.3204838838428259, + "reward_change_min": -0.5031619034707546, + "reward_change_std": 0.19439208041876554, + "reward_std": 0.47691015154123306, + "rewards/accuracy_reward": 0.20833334140479565, + "rewards/cosine_scaled_reward": -0.052808452397584915, + "step": 299 + }, + { + "clip_fraction": 0.0, + "completion_length": 3455.3958435058594, + "epoch": 0.34285714285714286, + "grad_norm": 0.059302717447280884, + "kl": 3.039836883544922e-05, + "lambda_div_used": 0.5659952610731125, + "learning_rate": 4.749540639777539e-07, + "loss": 0.0631, + "reward": -0.4138263203203678, + "reward_after_mean": -0.4138263203203678, + "reward_after_std": 0.35950295627117157, + "reward_before_mean": -0.14833886176347733, + "reward_before_std": 0.30170741491019726, + "reward_change_max": 0.0, + "reward_change_mean": -0.26548744924366474, + "reward_change_min": -0.45029690116643906, + "reward_change_std": 0.16043098457157612, + "reward_std": 0.35950295627117157, + "rewards/accuracy_reward": 0.0416666679084301, + "rewards/cosine_scaled_reward": -0.19000552780926228, + "step": 300 + }, + { + "clip_fraction": 0.0, + "completion_length": 2581.2917404174805, + "epoch": 0.344, + "grad_norm": 0.0829557478427887, + "kl": 1.7702579498291016e-05, + "lambda_div_used": 0.6020706444978714, + "learning_rate": 4.7185832004988133e-07, + "loss": 0.0033, + "reward": -0.2905968725681305, + "reward_after_mean": -0.2905968725681305, + "reward_after_std": 0.522302333265543, + "reward_before_mean": -0.03232934419065714, + "reward_before_std": 0.47291796933859587, + "reward_change_max": 0.0, + "reward_change_mean": -0.2582675274461508, + "reward_change_min": -0.4681916609406471, + "reward_change_std": 0.16694430727511644, + "reward_std": 0.5223023407161236, + "rewards/accuracy_reward": 0.10416666977107525, + "rewards/cosine_scaled_reward": -0.13649601582437754, + "step": 301 + }, + { + "clip_fraction": 0.0, + "completion_length": 2299.250026702881, + "epoch": 0.34514285714285714, + "grad_norm": 0.09259702265262604, + "kl": 2.473965287208557e-05, + "lambda_div_used": 0.6064160838723183, + "learning_rate": 4.68766384637248e-07, + "loss": 0.0049, + "reward": 0.1023632986471057, + "reward_after_mean": 0.1023632986471057, + "reward_after_std": 0.5881610047072172, + "reward_before_mean": 0.5643375236541033, + "reward_before_std": 0.48729276517406106, + "reward_change_max": 0.0, + "reward_change_mean": -0.46197423338890076, + "reward_change_min": -0.6793596595525742, + "reward_change_std": 0.2643775464966893, + "reward_std": 0.588161014020443, + "rewards/accuracy_reward": 0.39583334513008595, + "rewards/cosine_scaled_reward": 0.16850417526438832, + "step": 302 + }, + { + "clip_fraction": 0.0, + "completion_length": 2496.604202270508, + "epoch": 0.3462857142857143, + "grad_norm": 0.10342813283205032, + "kl": 2.822279930114746e-05, + "lambda_div_used": 0.6173663139343262, + "learning_rate": 4.656784084364238e-07, + "loss": -0.0508, + "reward": -0.21569269057363272, + "reward_after_mean": -0.21569269057363272, + "reward_after_std": 0.5887450613081455, + "reward_before_mean": 0.048040480352938175, + "reward_before_std": 0.5433634500950575, + "reward_change_max": 0.0, + "reward_change_mean": -0.2637331634759903, + "reward_change_min": -0.4554433934390545, + "reward_change_std": 0.17205023765563965, + "reward_std": 0.5887450724840164, + "rewards/accuracy_reward": 0.1458333358168602, + "rewards/cosine_scaled_reward": -0.09779285965487361, + "step": 303 + }, + { + "clip_fraction": 0.0, + "completion_length": 2828.375030517578, + "epoch": 0.3474285714285714, + "grad_norm": 0.06273287534713745, + "kl": 3.515370190143585e-05, + "lambda_div_used": 0.5850896239280701, + "learning_rate": 4.6259454195101267e-07, + "loss": 0.0395, + "reward": -0.351226020604372, + "reward_after_mean": -0.351226020604372, + "reward_after_std": 0.4407376032322645, + "reward_before_mean": -0.091004628688097, + "reward_before_std": 0.39247662480920553, + "reward_change_max": 0.0, + "reward_change_mean": -0.2602214030921459, + "reward_change_min": -0.4692201763391495, + "reward_change_std": 0.16711712814867496, + "reward_std": 0.4407376106828451, + "rewards/accuracy_reward": 0.0833333358168602, + "rewards/cosine_scaled_reward": -0.1743379645049572, + "step": 304 + }, + { + "clip_fraction": 0.0, + "completion_length": 2963.270835876465, + "epoch": 0.3485714285714286, + "grad_norm": 0.07068406045436859, + "kl": 3.521144390106201e-05, + "lambda_div_used": 0.5555498078465462, + "learning_rate": 4.59514935484316e-07, + "loss": 0.0077, + "reward": -0.44899775832891464, + "reward_after_mean": -0.44899775832891464, + "reward_after_std": 0.32930242642760277, + "reward_before_mean": -0.17746215965598822, + "reward_before_std": 0.2488851365633309, + "reward_change_max": 0.0, + "reward_change_mean": -0.27153559774160385, + "reward_change_min": -0.41173194721341133, + "reward_change_std": 0.14773181919008493, + "reward_std": 0.32930243387818336, + "rewards/accuracy_reward": 0.02083333395421505, + "rewards/cosine_scaled_reward": -0.19829548941925168, + "step": 305 + }, + { + "clip_fraction": 0.0, + "completion_length": 2542.8750228881836, + "epoch": 0.3497142857142857, + "grad_norm": 0.09160702675580978, + "kl": 2.777576446533203e-05, + "lambda_div_used": 0.5710241869091988, + "learning_rate": 4.5643973913200837e-07, + "loss": 0.001, + "reward": -0.28438286297023296, + "reward_after_mean": -0.28438286297023296, + "reward_after_std": 0.45469350554049015, + "reward_before_mean": 0.05048683221684769, + "reward_before_std": 0.3197702756151557, + "reward_change_max": 0.0, + "reward_change_mean": -0.33486970886588097, + "reward_change_min": -0.46306542679667473, + "reward_change_std": 0.1733616916462779, + "reward_std": 0.45469350926578045, + "rewards/accuracy_reward": 0.1666666679084301, + "rewards/cosine_scaled_reward": -0.11617984622716904, + "step": 306 + }, + { + "clip_fraction": 0.0, + "completion_length": 2358.9167098999023, + "epoch": 0.35085714285714287, + "grad_norm": 0.08308543264865875, + "kl": 2.5270506739616394e-05, + "lambda_div_used": 0.6283661872148514, + "learning_rate": 4.5336910277482155e-07, + "loss": 0.0233, + "reward": -0.023870151489973068, + "reward_after_mean": -0.023870151489973068, + "reward_after_std": 0.5968872811645269, + "reward_before_mean": 0.2986760139465332, + "reward_before_std": 0.602552474476397, + "reward_change_max": 0.0, + "reward_change_mean": -0.3225461672991514, + "reward_change_min": -0.5788566246628761, + "reward_change_std": 0.22817020770162344, + "reward_std": 0.5968873165547848, + "rewards/accuracy_reward": 0.2916666716337204, + "rewards/cosine_scaled_reward": 0.007009351626038551, + "step": 307 + }, + { + "clip_fraction": 0.0, + "completion_length": 3364.187530517578, + "epoch": 0.352, + "grad_norm": 0.04729427769780159, + "kl": 1.2226402759552002e-05, + "lambda_div_used": 0.5575956255197525, + "learning_rate": 4.503031760712397e-07, + "loss": -0.0147, + "reward": -0.364587739109993, + "reward_after_mean": -0.364587739109993, + "reward_after_std": 0.3172164801508188, + "reward_before_mean": -0.06630371138453484, + "reward_before_std": 0.25998193118721247, + "reward_change_max": 0.0, + "reward_change_mean": -0.29828402772545815, + "reward_change_min": -0.4606154337525368, + "reward_change_std": 0.17100436985492706, + "reward_std": 0.31721648946404457, + "rewards/accuracy_reward": 0.0833333358168602, + "rewards/cosine_scaled_reward": -0.14963705092668533, + "step": 308 + }, + { + "clip_fraction": 0.0, + "completion_length": 3246.750030517578, + "epoch": 0.35314285714285715, + "grad_norm": 0.054204076528549194, + "kl": 1.4858320355415344e-05, + "lambda_div_used": 0.6364770829677582, + "learning_rate": 4.4724210845020494e-07, + "loss": 0.0936, + "reward": -0.09930397570133209, + "reward_after_mean": -0.09930397570133209, + "reward_after_std": 0.6563027147203684, + "reward_before_mean": 0.18065341375768185, + "reward_before_std": 0.6305609010159969, + "reward_change_max": 0.0, + "reward_change_mean": -0.27995740808546543, + "reward_change_min": -0.5127242244780064, + "reward_change_std": 0.19019456766545773, + "reward_std": 0.6563027296215296, + "rewards/accuracy_reward": 0.2083333358168602, + "rewards/cosine_scaled_reward": -0.027679930441081524, + "step": 309 + }, + { + "clip_fraction": 0.0, + "completion_length": 2454.750015258789, + "epoch": 0.35428571428571426, + "grad_norm": 0.10188445448875427, + "kl": 3.9263395592570305e-05, + "lambda_div_used": 0.5863905549049377, + "learning_rate": 4.441860491038345e-07, + "loss": -0.0091, + "reward": -0.358197920024395, + "reward_after_mean": -0.358197920024395, + "reward_after_std": 0.4752412661910057, + "reward_before_mean": -0.10783447185531259, + "reward_before_std": 0.39106855262070894, + "reward_change_max": 0.0, + "reward_change_mean": -0.25036344304680824, + "reward_change_min": -0.36833222955465317, + "reward_change_std": 0.13517011515796185, + "reward_std": 0.47524126805365086, + "rewards/accuracy_reward": 0.06250000186264515, + "rewards/cosine_scaled_reward": -0.17033447965513915, + "step": 310 + }, + { + "clip_fraction": 0.0, + "completion_length": 2549.5000228881836, + "epoch": 0.3554285714285714, + "grad_norm": 0.07070475816726685, + "kl": 1.1476688086986542e-05, + "lambda_div_used": 0.6028245091438293, + "learning_rate": 4.4113514698014953e-07, + "loss": -0.0234, + "reward": -0.07257996033877134, + "reward_after_mean": -0.07257996033877134, + "reward_after_std": 0.5427698846906424, + "reward_before_mean": 0.32260693423449993, + "reward_before_std": 0.4735551681369543, + "reward_change_max": 0.0, + "reward_change_mean": -0.3951868824660778, + "reward_change_min": -0.6274620294570923, + "reward_change_std": 0.2447280865162611, + "reward_std": 0.5427699014544487, + "rewards/accuracy_reward": 0.3125000037252903, + "rewards/cosine_scaled_reward": 0.010106915608048439, + "step": 311 + }, + { + "clip_fraction": 0.0, + "completion_length": 2062.020851135254, + "epoch": 0.3565714285714286, + "grad_norm": 0.0864594504237175, + "kl": 2.2163614630699158e-05, + "lambda_div_used": 0.5457132831215858, + "learning_rate": 4.3808955077581546e-07, + "loss": 0.0122, + "reward": -0.03406492620706558, + "reward_after_mean": -0.03406492620706558, + "reward_after_std": 0.3851391561329365, + "reward_before_mean": 0.5170513242483139, + "reward_before_std": 0.20092127658426762, + "reward_change_max": 0.0, + "reward_change_mean": -0.5511162541806698, + "reward_change_min": -0.7572819888591766, + "reward_change_std": 0.2855915669351816, + "reward_std": 0.38513917103409767, + "rewards/accuracy_reward": 0.375, + "rewards/cosine_scaled_reward": 0.1420513167977333, + "step": 312 + }, + { + "clip_fraction": 0.0, + "completion_length": 2939.958351135254, + "epoch": 0.3577142857142857, + "grad_norm": 0.06731264293193817, + "kl": 2.4283304810523987e-05, + "lambda_div_used": 0.5877447426319122, + "learning_rate": 4.350494089288943e-07, + "loss": -0.0439, + "reward": -0.15467195864766836, + "reward_after_mean": -0.15467195864766836, + "reward_after_std": 0.522945849224925, + "reward_before_mean": 0.20540180057287216, + "reward_before_std": 0.40051606576889753, + "reward_change_max": 0.0, + "reward_change_mean": -0.3600737862288952, + "reward_change_min": -0.5124829597771168, + "reward_change_std": 0.19857864920049906, + "reward_std": 0.5229458846151829, + "rewards/accuracy_reward": 0.2291666679084301, + "rewards/cosine_scaled_reward": -0.02376485476270318, + "step": 313 + }, + { + "clip_fraction": 0.0, + "completion_length": 2541.18754196167, + "epoch": 0.3588571428571429, + "grad_norm": 0.08012691885232925, + "kl": 3.474205732345581e-05, + "lambda_div_used": 0.607881672680378, + "learning_rate": 4.3201486961161093e-07, + "loss": 0.0168, + "reward": -0.20861644856631756, + "reward_after_mean": -0.20861644856631756, + "reward_after_std": 0.5089061111211777, + "reward_before_mean": 0.08002243563532829, + "reward_before_std": 0.49900877848267555, + "reward_change_max": 0.0, + "reward_change_mean": -0.28863888792693615, + "reward_change_min": -0.548176895827055, + "reward_change_std": 0.2009973768144846, + "reward_std": 0.5089061167091131, + "rewards/accuracy_reward": 0.1666666716337204, + "rewards/cosine_scaled_reward": -0.08664422971196473, + "step": 314 + }, + { + "clip_fraction": 0.0, + "completion_length": 3124.3333740234375, + "epoch": 0.36, + "grad_norm": 0.0558871328830719, + "kl": 1.9735191017389297e-05, + "lambda_div_used": 0.5734824016690254, + "learning_rate": 4.2898608072313045e-07, + "loss": 0.0494, + "reward": -0.1729298224672675, + "reward_after_mean": -0.1729298224672675, + "reward_after_std": 0.45045475475490093, + "reward_before_mean": 0.20758753083646297, + "reward_before_std": 0.33522335812449455, + "reward_change_max": 0.0, + "reward_change_mean": -0.3805173486471176, + "reward_change_min": -0.5440769977867603, + "reward_change_std": 0.20876472163945436, + "reward_std": 0.4504547640681267, + "rewards/accuracy_reward": 0.22916666977107525, + "rewards/cosine_scaled_reward": -0.02157914894632995, + "step": 315 + }, + { + "clip_fraction": 0.0, + "completion_length": 3509.5416870117188, + "epoch": 0.36114285714285715, + "grad_norm": 0.05331319198012352, + "kl": 2.981536090373993e-05, + "lambda_div_used": 0.5893311724066734, + "learning_rate": 4.2596318988235037e-07, + "loss": 0.0135, + "reward": -0.22396450489759445, + "reward_after_mean": -0.22396450489759445, + "reward_after_std": 0.43868801929056644, + "reward_before_mean": 0.07456053979694843, + "reward_before_std": 0.414919788017869, + "reward_change_max": 0.0, + "reward_change_mean": -0.29852502048015594, + "reward_change_min": -0.4823254942893982, + "reward_change_std": 0.1912739286199212, + "reward_std": 0.43868803791701794, + "rewards/accuracy_reward": 0.1666666716337204, + "rewards/cosine_scaled_reward": -0.09210613742470741, + "step": 316 + }, + { + "clip_fraction": 0.0, + "completion_length": 3099.5833435058594, + "epoch": 0.36228571428571427, + "grad_norm": 0.06200157478451729, + "kl": 3.091990947723389e-05, + "lambda_div_used": 0.5825675800442696, + "learning_rate": 4.2294634442070553e-07, + "loss": 0.0437, + "reward": -0.35854507237672806, + "reward_after_mean": -0.35854507237672806, + "reward_after_std": 0.43180895783007145, + "reward_before_mean": -0.102005859836936, + "reward_before_std": 0.3769839182496071, + "reward_change_max": 0.0, + "reward_change_mean": -0.25653921999037266, + "reward_change_min": -0.4170425795018673, + "reward_change_std": 0.15367608424276114, + "reward_std": 0.4318089634180069, + "rewards/accuracy_reward": 0.08333333395421505, + "rewards/cosine_scaled_reward": -0.18533920124173164, + "step": 317 + }, + { + "clip_fraction": 0.0, + "completion_length": 2281.000068664551, + "epoch": 0.36342857142857143, + "grad_norm": 0.0932127833366394, + "kl": 3.602728247642517e-05, + "lambda_div_used": 0.6223095878958702, + "learning_rate": 4.1993569137498776e-07, + "loss": 0.0631, + "reward": -0.07363937655463815, + "reward_after_mean": -0.07363937655463815, + "reward_after_std": 0.5691333152353764, + "reward_before_mean": 0.24711408838629723, + "reward_before_std": 0.5656038168817759, + "reward_change_max": 0.0, + "reward_change_mean": -0.3207534924149513, + "reward_change_min": -0.5814453661441803, + "reward_change_std": 0.2219030074775219, + "reward_std": 0.569133322685957, + "rewards/accuracy_reward": 0.2500000074505806, + "rewards/cosine_scaled_reward": -0.0028859074227511883, + "step": 318 + }, + { + "clip_fraction": 0.0, + "completion_length": 2854.5625076293945, + "epoch": 0.36457142857142855, + "grad_norm": 0.08494079858064651, + "kl": 2.7507543563842773e-05, + "lambda_div_used": 0.5380512997508049, + "learning_rate": 4.1693137748017915e-07, + "loss": -0.0389, + "reward": -0.4965253435075283, + "reward_after_mean": -0.4965253435075283, + "reward_after_std": 0.2333353590220213, + "reward_before_mean": -0.2120634987950325, + "reward_before_std": 0.16711975168436766, + "reward_change_max": 0.0, + "reward_change_mean": -0.28446184657514095, + "reward_change_min": -0.4124374948441982, + "reward_change_std": 0.15511877462267876, + "reward_std": 0.2333353627473116, + "rewards/accuracy_reward": 0.0, + "rewards/cosine_scaled_reward": -0.2120634950697422, + "step": 319 + }, + { + "clip_fraction": 0.0, + "completion_length": 1964.8958740234375, + "epoch": 0.3657142857142857, + "grad_norm": 0.09068436175584793, + "kl": 3.33394855260849e-05, + "lambda_div_used": 0.6160280704498291, + "learning_rate": 4.1393354916230005e-07, + "loss": 0.0117, + "reward": -0.19740011962130666, + "reward_after_mean": -0.19740011962130666, + "reward_after_std": 0.5963947810232639, + "reward_before_mean": 0.0693174353800714, + "reward_before_std": 0.5336382519453764, + "reward_change_max": 0.0, + "reward_change_mean": -0.26671755872666836, + "reward_change_min": -0.4143567681312561, + "reward_change_std": 0.15817437414079905, + "reward_std": 0.5963947977870703, + "rewards/accuracy_reward": 0.12500000186264515, + "rewards/cosine_scaled_reward": -0.05568256159313023, + "step": 320 + }, + { + "clip_fraction": 0.0, + "completion_length": 1974.8958778381348, + "epoch": 0.3668571428571429, + "grad_norm": 0.0693729817867279, + "kl": 1.574307680130005e-05, + "lambda_div_used": 0.6290735602378845, + "learning_rate": 4.1094235253127374e-07, + "loss": 0.0387, + "reward": 0.11023577488958836, + "reward_after_mean": 0.11023577488958836, + "reward_after_std": 0.6442763805389404, + "reward_before_mean": 0.5218545235693455, + "reward_before_std": 0.6013733670115471, + "reward_change_max": 0.0, + "reward_change_mean": -0.4116187300533056, + "reward_change_min": -0.6836260408163071, + "reward_change_std": 0.2705372450873256, + "reward_std": 0.6442763898521662, + "rewards/accuracy_reward": 0.3958333395421505, + "rewards/cosine_scaled_reward": 0.12602117005735636, + "step": 321 + }, + { + "clip_fraction": 0.0, + "completion_length": 2837.833381652832, + "epoch": 0.368, + "grad_norm": 0.10218925029039383, + "kl": 4.9501657485961914e-05, + "lambda_div_used": 0.6059064492583275, + "learning_rate": 4.079579333738039e-07, + "loss": -0.0178, + "reward": -0.2954628551378846, + "reward_after_mean": -0.2954628551378846, + "reward_after_std": 0.5665331184864044, + "reward_before_mean": -0.04816420469433069, + "reward_before_std": 0.48093545995652676, + "reward_change_max": 0.0, + "reward_change_mean": -0.24729863554239273, + "reward_change_min": -0.3516420107334852, + "reward_change_std": 0.13184416200965643, + "reward_std": 0.5665331222116947, + "rewards/accuracy_reward": 0.0833333358168602, + "rewards/cosine_scaled_reward": -0.13149754563346505, + "step": 322 + }, + { + "clip_fraction": 0.0, + "completion_length": 3015.666717529297, + "epoch": 0.36914285714285716, + "grad_norm": 0.0596209317445755, + "kl": 1.7877668142318726e-05, + "lambda_div_used": 0.5565166473388672, + "learning_rate": 4.0498043714627006e-07, + "loss": 0.0465, + "reward": -0.16177266091108322, + "reward_after_mean": -0.16177266091108322, + "reward_after_std": 0.42436218820512295, + "reward_before_mean": 0.2816486116498709, + "reward_before_std": 0.2578566027805209, + "reward_change_max": 0.0, + "reward_change_mean": -0.4434212874621153, + "reward_change_min": -0.6335049495100975, + "reward_change_std": 0.23841418512165546, + "reward_std": 0.4243622049689293, + "rewards/accuracy_reward": 0.2916666679084301, + "rewards/cosine_scaled_reward": -0.010018057189881802, + "step": 323 + }, + { + "clip_fraction": 0.0, + "completion_length": 2751.6458435058594, + "epoch": 0.3702857142857143, + "grad_norm": 0.10376841574907303, + "kl": 2.1306797862052917e-05, + "lambda_div_used": 0.5670785158872604, + "learning_rate": 4.020100089676376e-07, + "loss": -0.0038, + "reward": -0.10797288408502936, + "reward_after_mean": -0.10797288408502936, + "reward_after_std": 0.4300071895122528, + "reward_before_mean": 0.3295288155786693, + "reward_before_std": 0.3011532872915268, + "reward_change_max": 0.0, + "reward_change_mean": -0.43750171549618244, + "reward_change_min": -0.6185614429414272, + "reward_change_std": 0.2333526872098446, + "reward_std": 0.43000719882547855, + "rewards/accuracy_reward": 0.25000000558793545, + "rewards/cosine_scaled_reward": 0.07952881557866931, + "step": 324 + }, + { + "clip_fraction": 0.0, + "completion_length": 2943.8958587646484, + "epoch": 0.37142857142857144, + "grad_norm": 0.07239633053541183, + "kl": 9.275972843170166e-06, + "lambda_div_used": 0.6356078162789345, + "learning_rate": 3.9904679361238526e-07, + "loss": 0.0667, + "reward": -0.050798285752534866, + "reward_after_mean": -0.050798285752534866, + "reward_after_std": 0.6440786644816399, + "reward_before_mean": 0.24087253957986832, + "reward_before_std": 0.6331603992730379, + "reward_change_max": 0.0, + "reward_change_mean": -0.2916708290576935, + "reward_change_min": -0.4932599440217018, + "reward_change_std": 0.20057004038244486, + "reward_std": 0.6440786886960268, + "rewards/accuracy_reward": 0.2708333395421505, + "rewards/cosine_scaled_reward": -0.02996080555021763, + "step": 325 + }, + { + "clip_fraction": 0.0, + "completion_length": 2375.645851135254, + "epoch": 0.37257142857142855, + "grad_norm": 0.09299691766500473, + "kl": 2.9705464839935303e-05, + "lambda_div_used": 0.5330315083265305, + "learning_rate": 3.9609093550344907e-07, + "loss": -0.0979, + "reward": -0.20862240344285965, + "reward_after_mean": -0.20862240344285965, + "reward_after_std": 0.3296848703175783, + "reward_before_mean": 0.2517512815538794, + "reward_before_std": 0.1450797226279974, + "reward_change_max": 0.0, + "reward_change_mean": -0.460373692214489, + "reward_change_min": -0.6264396719634533, + "reward_change_std": 0.23280893173068762, + "reward_std": 0.3296848740428686, + "rewards/accuracy_reward": 0.25, + "rewards/cosine_scaled_reward": 0.001751287141814828, + "step": 326 + }, + { + "clip_fraction": 0.0, + "completion_length": 2810.187511444092, + "epoch": 0.3737142857142857, + "grad_norm": 0.06210591271519661, + "kl": 8.400529623031616e-06, + "lambda_div_used": 0.5783005133271217, + "learning_rate": 3.931425787051832e-07, + "loss": -0.0483, + "reward": -0.11793380603194237, + "reward_after_mean": -0.11793380603194237, + "reward_after_std": 0.45353105291724205, + "reward_before_mean": 0.2935123089700937, + "reward_before_std": 0.3566841436550021, + "reward_change_max": 0.0, + "reward_change_mean": -0.41144610941410065, + "reward_change_min": -0.6089312434196472, + "reward_change_std": 0.23675687983632088, + "reward_std": 0.4535310585051775, + "rewards/accuracy_reward": 0.2708333395421505, + "rewards/cosine_scaled_reward": 0.022678975947201252, + "step": 327 + }, + { + "clip_fraction": 0.0, + "completion_length": 3518.8958435058594, + "epoch": 0.37485714285714283, + "grad_norm": 0.05192619562149048, + "kl": 2.485513687133789e-05, + "lambda_div_used": 0.5715365409851074, + "learning_rate": 3.902018669163384e-07, + "loss": 0.0155, + "reward": -0.4497411046177149, + "reward_after_mean": -0.4497411046177149, + "reward_after_std": 0.40177739411592484, + "reward_before_mean": -0.21567542850971222, + "reward_before_std": 0.32284149527549744, + "reward_change_max": 0.0, + "reward_change_mean": -0.23406569100916386, + "reward_change_min": -0.33814629539847374, + "reward_change_std": 0.12489900179207325, + "reward_std": 0.40177739597857, + "rewards/accuracy_reward": 0.0416666679084301, + "rewards/cosine_scaled_reward": -0.2573420889675617, + "step": 328 + }, + { + "clip_fraction": 0.0, + "completion_length": 2243.0416946411133, + "epoch": 0.376, + "grad_norm": 0.10950610786676407, + "kl": 2.469681203365326e-05, + "lambda_div_used": 0.6176783889532089, + "learning_rate": 3.872689434630585e-07, + "loss": -0.0562, + "reward": -0.013319691643118858, + "reward_after_mean": -0.013319691643118858, + "reward_after_std": 0.5674807205796242, + "reward_before_mean": 0.3346351385116577, + "reward_before_std": 0.5435493532568216, + "reward_change_max": 0.0, + "reward_change_mean": -0.34795483760535717, + "reward_change_min": -0.5437430925667286, + "reward_change_std": 0.22110824659466743, + "reward_std": 0.5674807410687208, + "rewards/accuracy_reward": 0.27083334140479565, + "rewards/cosine_scaled_reward": 0.06380179291591048, + "step": 329 + }, + { + "clip_fraction": 0.0, + "completion_length": 2141.479190826416, + "epoch": 0.37714285714285717, + "grad_norm": 0.10051169991493225, + "kl": 4.89354133605957e-05, + "lambda_div_used": 0.5594945177435875, + "learning_rate": 3.843439512918949e-07, + "loss": -0.0064, + "reward": -0.43674849811941385, + "reward_after_mean": -0.43674849811941385, + "reward_after_std": 0.32430145144462585, + "reward_before_mean": -0.17029727809131145, + "reward_before_std": 0.2681279256939888, + "reward_change_max": 0.0, + "reward_change_mean": -0.266451220959425, + "reward_change_min": -0.44791119545698166, + "reward_change_std": 0.1575082140043378, + "reward_std": 0.32430145516991615, + "rewards/accuracy_reward": 0.0416666679084301, + "rewards/cosine_scaled_reward": -0.2119639493757859, + "step": 330 + }, + { + "clip_fraction": 0.0, + "completion_length": 2564.604202270508, + "epoch": 0.3782857142857143, + "grad_norm": 0.12382425367832184, + "kl": 4.225596785545349e-05, + "lambda_div_used": 0.5574543103575706, + "learning_rate": 3.8142703296283953e-07, + "loss": 0.0413, + "reward": -0.41342686861753464, + "reward_after_mean": -0.41342686861753464, + "reward_after_std": 0.33145094849169254, + "reward_before_mean": -0.13134576752781868, + "reward_before_std": 0.25930391903966665, + "reward_change_max": 0.0, + "reward_change_mean": -0.2820810880511999, + "reward_change_min": -0.4496995583176613, + "reward_change_std": 0.16215670108795166, + "reward_std": 0.33145095966756344, + "rewards/accuracy_reward": 0.0416666679084301, + "rewards/cosine_scaled_reward": -0.17301243357360363, + "step": 331 + }, + { + "clip_fraction": 0.0, + "completion_length": 2420.6666870117188, + "epoch": 0.37942857142857145, + "grad_norm": 0.07938601076602936, + "kl": 2.1241605281829834e-05, + "lambda_div_used": 0.5416555106639862, + "learning_rate": 3.785183306423767e-07, + "loss": 0.0287, + "reward": -0.21761326864361763, + "reward_after_mean": -0.21761326864361763, + "reward_after_std": 0.34953613951802254, + "reward_before_mean": 0.2352729644626379, + "reward_before_std": 0.18298226408660412, + "reward_change_max": 0.0, + "reward_change_mean": -0.45288625732064247, + "reward_change_min": -0.6215734221041203, + "reward_change_std": 0.2345103258267045, + "reward_std": 0.34953614324331284, + "rewards/accuracy_reward": 0.25, + "rewards/cosine_scaled_reward": -0.014727018773555756, + "step": 332 + }, + { + "clip_fraction": 0.0, + "completion_length": 2350.6875381469727, + "epoch": 0.38057142857142856, + "grad_norm": 0.08657004684209824, + "kl": 2.1725893020629883e-05, + "lambda_div_used": 0.6729468256235123, + "learning_rate": 3.7561798609655373e-07, + "loss": 0.0369, + "reward": -0.01155824027955532, + "reward_after_mean": -0.01155824027955532, + "reward_after_std": 0.8104779217392206, + "reward_before_mean": 0.23989354074001312, + "reward_before_std": 0.8117707390338182, + "reward_change_max": 0.0, + "reward_change_mean": -0.2514517717063427, + "reward_change_min": -0.5040576457977295, + "reward_change_std": 0.19348772894591093, + "reward_std": 0.810477938503027, + "rewards/accuracy_reward": 0.2708333395421505, + "rewards/cosine_scaled_reward": -0.030939804390072823, + "step": 333 + }, + { + "clip_fraction": 0.0, + "completion_length": 3491.6041870117188, + "epoch": 0.38171428571428573, + "grad_norm": 0.05208956450223923, + "kl": 1.1973665095865726e-05, + "lambda_div_used": 0.5869207382202148, + "learning_rate": 3.72726140684072e-07, + "loss": -0.009, + "reward": -0.2944636158645153, + "reward_after_mean": -0.2944636158645153, + "reward_after_std": 0.43844909220933914, + "reward_before_mean": -0.013160821050405502, + "reward_before_std": 0.40495526185259223, + "reward_change_max": 0.0, + "reward_change_mean": -0.2813027612864971, + "reward_change_min": -0.4717176593840122, + "reward_change_std": 0.1835445323958993, + "reward_std": 0.4384490940719843, + "rewards/accuracy_reward": 0.125, + "rewards/cosine_scaled_reward": -0.13816083781421185, + "step": 334 + }, + { + "clip_fraction": 0.0, + "completion_length": 2589.0833892822266, + "epoch": 0.38285714285714284, + "grad_norm": 0.0683097094297409, + "kl": 1.9595026969909668e-05, + "lambda_div_used": 0.6027273386716843, + "learning_rate": 3.6984293534939737e-07, + "loss": -0.0488, + "reward": 0.005996011197566986, + "reward_after_mean": 0.005996011197566986, + "reward_after_std": 0.5422599408775568, + "reward_before_mean": 0.41999348998069763, + "reward_before_std": 0.4701922629028559, + "reward_change_max": 0.0, + "reward_change_mean": -0.41399746760725975, + "reward_change_min": -0.623339332640171, + "reward_change_std": 0.24341293703764677, + "reward_std": 0.5422599650919437, + "rewards/accuracy_reward": 0.33333334140479565, + "rewards/cosine_scaled_reward": 0.08666014112532139, + "step": 335 + }, + { + "clip_fraction": 0.0, + "completion_length": 3162.1458740234375, + "epoch": 0.384, + "grad_norm": 0.067026786506176, + "kl": 2.0024715922772884e-05, + "lambda_div_used": 0.5950900241732597, + "learning_rate": 3.6696851061588994e-07, + "loss": 0.0138, + "reward": -0.16571878641843796, + "reward_after_mean": -0.16571878641843796, + "reward_after_std": 0.4802239239215851, + "reward_before_mean": 0.14471458829939365, + "reward_before_std": 0.4368621027097106, + "reward_change_max": 0.0, + "reward_change_mean": -0.31043340265750885, + "reward_change_min": -0.45523249730467796, + "reward_change_std": 0.18632046319544315, + "reward_std": 0.48022393323481083, + "rewards/accuracy_reward": 0.20833334140479565, + "rewards/cosine_scaled_reward": -0.06361875729635358, + "step": 336 + }, + { + "clip_fraction": 0.0, + "completion_length": 3228.916717529297, + "epoch": 0.3851428571428571, + "grad_norm": 0.05528295040130615, + "kl": 1.9825994968414307e-05, + "lambda_div_used": 0.6006625667214394, + "learning_rate": 3.641030065789562e-07, + "loss": 0.0552, + "reward": -0.2433483125641942, + "reward_after_mean": -0.2433483125641942, + "reward_after_std": 0.5013067033141851, + "reward_before_mean": 0.03825543075799942, + "reward_before_std": 0.4624515902251005, + "reward_change_max": 0.0, + "reward_change_mean": -0.28160375356674194, + "reward_change_min": -0.4847262054681778, + "reward_change_std": 0.17836903873831034, + "reward_std": 0.501306714490056, + "rewards/accuracy_reward": 0.14583333767950535, + "rewards/cosine_scaled_reward": -0.10757789574563503, + "step": 337 + }, + { + "clip_fraction": 0.0, + "completion_length": 2393.645866394043, + "epoch": 0.3862857142857143, + "grad_norm": 0.10292567312717438, + "kl": 3.5159289836883545e-05, + "lambda_div_used": 0.6184682846069336, + "learning_rate": 3.612465628992203e-07, + "loss": 0.1082, + "reward": -0.010080082342028618, + "reward_after_mean": -0.010080082342028618, + "reward_after_std": 0.6298389285802841, + "reward_before_mean": 0.3685674872249365, + "reward_before_std": 0.546540604904294, + "reward_change_max": 0.0, + "reward_change_mean": -0.3786476030945778, + "reward_change_min": -0.5825354494154453, + "reward_change_std": 0.2281006295233965, + "reward_std": 0.6298389509320259, + "rewards/accuracy_reward": 0.3125000037252903, + "rewards/cosine_scaled_reward": 0.05606749979779124, + "step": 338 + }, + { + "clip_fraction": 0.0, + "completion_length": 3026.9583892822266, + "epoch": 0.38742857142857146, + "grad_norm": 0.0688738226890564, + "kl": 2.4460256099700928e-05, + "lambda_div_used": 0.5621443763375282, + "learning_rate": 3.5839931879571725e-07, + "loss": -0.0178, + "reward": -0.32898029685020447, + "reward_after_mean": -0.32898029685020447, + "reward_after_std": 0.3227359801530838, + "reward_before_mean": -0.01589403674006462, + "reward_before_std": 0.2781273443251848, + "reward_change_max": 0.0, + "reward_change_mean": -0.31308628246188164, + "reward_change_min": -0.48185204714536667, + "reward_change_std": 0.1813461184501648, + "reward_std": 0.3227359913289547, + "rewards/accuracy_reward": 0.1041666716337204, + "rewards/cosine_scaled_reward": -0.12006070092320442, + "step": 339 + }, + { + "clip_fraction": 0.0, + "completion_length": 2447.937515258789, + "epoch": 0.38857142857142857, + "grad_norm": 0.10290948301553726, + "kl": 2.363821113249287e-05, + "lambda_div_used": 0.5592405423521996, + "learning_rate": 3.555614130391079e-07, + "loss": -0.0046, + "reward": -0.42997122183442116, + "reward_after_mean": -0.42997122183442116, + "reward_after_std": 0.33643546886742115, + "reward_before_mean": -0.1512418081983924, + "reward_before_std": 0.2663228642195463, + "reward_change_max": 0.0, + "reward_change_mean": -0.2787294201552868, + "reward_change_min": -0.4129678010940552, + "reward_change_std": 0.1523351836949587, + "reward_std": 0.3364354781806469, + "rewards/accuracy_reward": 0.02083333395421505, + "rewards/cosine_scaled_reward": -0.17207514215260744, + "step": 340 + }, + { + "clip_fraction": 0.0, + "completion_length": 2488.9583587646484, + "epoch": 0.38971428571428574, + "grad_norm": 0.061042170971632004, + "kl": 1.909211277961731e-05, + "lambda_div_used": 0.5834595933556557, + "learning_rate": 3.5273298394491515e-07, + "loss": -0.0023, + "reward": 0.009830114664509892, + "reward_after_mean": 0.009830114664509892, + "reward_after_std": 0.5199873205274343, + "reward_before_mean": 0.4715769328176975, + "reward_before_std": 0.38039534725248814, + "reward_change_max": 0.0, + "reward_change_mean": -0.46174679696559906, + "reward_change_min": -0.6357490979135036, + "reward_change_std": 0.2529716519638896, + "reward_std": 0.5199873335659504, + "rewards/accuracy_reward": 0.33333334140479565, + "rewards/cosine_scaled_reward": 0.13824361143633723, + "step": 341 + }, + { + "clip_fraction": 0.0, + "completion_length": 3020.520854949951, + "epoch": 0.39085714285714285, + "grad_norm": 0.07325860857963562, + "kl": 2.4762004613876343e-05, + "lambda_div_used": 0.5985761880874634, + "learning_rate": 3.4991416936678276e-07, + "loss": 0.0007, + "reward": -0.06264204811304808, + "reward_after_mean": -0.06264204811304808, + "reward_after_std": 0.5387034490704536, + "reward_before_mean": 0.33080895524472, + "reward_before_std": 0.45346040464937687, + "reward_change_max": 0.0, + "reward_change_mean": -0.3934510052204132, + "reward_change_min": -0.5952773988246918, + "reward_change_std": 0.23232300952076912, + "reward_std": 0.5387034583836794, + "rewards/accuracy_reward": 0.29166666977107525, + "rewards/cosine_scaled_reward": 0.03914228640496731, + "step": 342 + }, + { + "clip_fraction": 0.0, + "completion_length": 3405.1875610351562, + "epoch": 0.392, + "grad_norm": 0.04665439575910568, + "kl": 2.7433037757873535e-05, + "lambda_div_used": 0.6150833070278168, + "learning_rate": 3.471051066897562e-07, + "loss": 0.0509, + "reward": -0.1582264108583331, + "reward_after_mean": -0.1582264108583331, + "reward_after_std": 0.5790915079414845, + "reward_before_mean": 0.13680532574653625, + "reward_before_std": 0.5264813583344221, + "reward_change_max": 0.0, + "reward_change_mean": -0.29503174126148224, + "reward_change_min": -0.4708465002477169, + "reward_change_std": 0.1767220702022314, + "reward_std": 0.5790915191173553, + "rewards/accuracy_reward": 0.1875000074505806, + "rewards/cosine_scaled_reward": -0.050694676116108894, + "step": 343 + }, + { + "clip_fraction": 0.0, + "completion_length": 2360.875045776367, + "epoch": 0.3931428571428571, + "grad_norm": 0.07271434366703033, + "kl": 1.041218638420105e-05, + "lambda_div_used": 0.5770246461033821, + "learning_rate": 3.4430593282358777e-07, + "loss": -0.0008, + "reward": 0.04454847797751427, + "reward_after_mean": 0.04454847797751427, + "reward_after_std": 0.5018230397254229, + "reward_before_mean": 0.5492894258350134, + "reward_before_std": 0.35145203582942486, + "reward_change_max": 0.0, + "reward_change_mean": -0.5047409404069185, + "reward_change_min": -0.7455123476684093, + "reward_change_std": 0.2790954224765301, + "reward_std": 0.5018230620771646, + "rewards/accuracy_reward": 0.4375, + "rewards/cosine_scaled_reward": 0.1117894072085619, + "step": 344 + }, + { + "clip_fraction": 0.0, + "completion_length": 3030.0833740234375, + "epoch": 0.3942857142857143, + "grad_norm": 0.0703606829047203, + "kl": 3.1925737857818604e-05, + "lambda_div_used": 0.5804503262042999, + "learning_rate": 3.4151678419606233e-07, + "loss": 0.0646, + "reward": -0.22463034093379974, + "reward_after_mean": -0.22463034093379974, + "reward_after_std": 0.41369511373341084, + "reward_before_mean": 0.09700941108167171, + "reward_before_std": 0.37086532823741436, + "reward_change_max": 0.0, + "reward_change_mean": -0.32163975574076176, + "reward_change_min": -0.5076877251267433, + "reward_change_std": 0.1956562791019678, + "reward_std": 0.4136951379477978, + "rewards/accuracy_reward": 0.1666666716337204, + "rewards/cosine_scaled_reward": -0.06965726800262928, + "step": 345 + }, + { + "clip_fraction": 0.0, + "completion_length": 3186.6458740234375, + "epoch": 0.3954285714285714, + "grad_norm": 0.05292130261659622, + "kl": 1.4215707778930664e-05, + "lambda_div_used": 0.5958857163786888, + "learning_rate": 3.387377967463493e-07, + "loss": 0.0331, + "reward": -0.24454469978809357, + "reward_after_mean": -0.24454469978809357, + "reward_after_std": 0.48920249938964844, + "reward_before_mean": 0.04633236164227128, + "reward_before_std": 0.44041235372424126, + "reward_change_max": 0.0, + "reward_change_mean": -0.29087705351412296, + "reward_change_min": -0.4697858430445194, + "reward_change_std": 0.179568306542933, + "reward_std": 0.4892025087028742, + "rewards/accuracy_reward": 0.16666667349636555, + "rewards/cosine_scaled_reward": -0.12033431546296924, + "step": 346 + }, + { + "clip_fraction": 0.0, + "completion_length": 3055.4583587646484, + "epoch": 0.3965714285714286, + "grad_norm": 0.07354738563299179, + "kl": 6.0675665736198425e-06, + "lambda_div_used": 0.5582710355520248, + "learning_rate": 3.359691059183761e-07, + "loss": 0.0132, + "reward": -0.42621116526424885, + "reward_after_mean": -0.42621116526424885, + "reward_after_std": 0.3322943150997162, + "reward_before_mean": -0.15516536496579647, + "reward_before_std": 0.2633455842733383, + "reward_change_max": 0.0, + "reward_change_mean": -0.2710457965731621, + "reward_change_min": -0.4307188205420971, + "reward_change_std": 0.15532648842781782, + "reward_std": 0.33229432441294193, + "rewards/accuracy_reward": 0.0416666679084301, + "rewards/cosine_scaled_reward": -0.19683203659951687, + "step": 347 + }, + { + "clip_fraction": 0.0, + "completion_length": 2852.4583435058594, + "epoch": 0.3977142857142857, + "grad_norm": 0.09605983644723892, + "kl": 3.167241811752319e-05, + "lambda_div_used": 0.5788434967398643, + "learning_rate": 3.3321084665422803e-07, + "loss": -0.0017, + "reward": -0.2087371125817299, + "reward_after_mean": -0.2087371125817299, + "reward_after_std": 0.40125221759080887, + "reward_before_mean": 0.12491290923207998, + "reward_before_std": 0.36025606002658606, + "reward_change_max": 0.0, + "reward_change_mean": -0.3336500097066164, + "reward_change_min": -0.5146553069353104, + "reward_change_std": 0.20004253275692463, + "reward_std": 0.40125224366784096, + "rewards/accuracy_reward": 0.1666666716337204, + "rewards/cosine_scaled_reward": -0.04175376985222101, + "step": 348 + }, + { + "clip_fraction": 0.0, + "completion_length": 2964.875015258789, + "epoch": 0.39885714285714285, + "grad_norm": 0.06869405508041382, + "kl": 1.731887459754944e-05, + "lambda_div_used": 0.5606642514467239, + "learning_rate": 3.3046315338757026e-07, + "loss": 0.0031, + "reward": -0.2807905152440071, + "reward_after_mean": -0.2807905152440071, + "reward_after_std": 0.39816635474562645, + "reward_before_mean": 0.08202904835343361, + "reward_before_std": 0.2763519547879696, + "reward_change_max": 0.0, + "reward_change_mean": -0.36281956918537617, + "reward_change_min": -0.54488330706954, + "reward_change_std": 0.20150058157742023, + "reward_std": 0.39816636219620705, + "rewards/accuracy_reward": 0.1666666679084301, + "rewards/cosine_scaled_reward": -0.0846376121044159, + "step": 349 + }, + { + "clip_fraction": 0.0, + "completion_length": 2521.9791870117188, + "epoch": 0.4, + "grad_norm": 0.10303740203380585, + "kl": 2.6671215891838074e-05, + "lambda_div_used": 0.6121688261628151, + "learning_rate": 3.2772616003709616e-07, + "loss": -0.0711, + "reward": -0.04820730444043875, + "reward_after_mean": -0.04820730444043875, + "reward_after_std": 0.5604431573301554, + "reward_before_mean": 0.287683189380914, + "reward_before_std": 0.5214535812847316, + "reward_change_max": 0.0, + "reward_change_mean": -0.3358904607594013, + "reward_change_min": -0.5252225995063782, + "reward_change_std": 0.21241459622979164, + "reward_std": 0.5604431666433811, + "rewards/accuracy_reward": 0.27083334140479565, + "rewards/cosine_scaled_reward": 0.016849845182150602, + "step": 350 + }, + { + "clip_fraction": 0.0, + "completion_length": 3268.250030517578, + "epoch": 0.40114285714285713, + "grad_norm": 0.04833042621612549, + "kl": 2.4568289518356323e-05, + "lambda_div_used": 0.5601532310247421, + "learning_rate": 3.250000000000001e-07, + "loss": 0.0519, + "reward": -0.250777292996645, + "reward_after_mean": -0.250777292996645, + "reward_after_std": 0.38171103224158287, + "reward_before_mean": 0.1175131555646658, + "reward_before_std": 0.2718004435300827, + "reward_change_max": 0.0, + "reward_change_mean": -0.36829047463834286, + "reward_change_min": -0.5422449931502342, + "reward_change_std": 0.20415859669446945, + "reward_std": 0.3817110415548086, + "rewards/accuracy_reward": 0.1875, + "rewards/cosine_scaled_reward": -0.06998682580888271, + "step": 351 + }, + { + "clip_fraction": 0.0, + "completion_length": 2589.333366394043, + "epoch": 0.4022857142857143, + "grad_norm": 0.06552080065011978, + "kl": 1.905485987663269e-05, + "lambda_div_used": 0.6104472130537033, + "learning_rate": 3.222848061454764e-07, + "loss": 0.0025, + "reward": -0.09347447147592902, + "reward_after_mean": -0.09347447147592902, + "reward_after_std": 0.6156985405832529, + "reward_before_mean": 0.26158976616716245, + "reward_before_std": 0.5061299707740545, + "reward_change_max": 0.0, + "reward_change_mean": -0.3550642393529415, + "reward_change_min": -0.5232127867639065, + "reward_change_std": 0.1957317991182208, + "reward_std": 0.6156985703855753, + "rewards/accuracy_reward": 0.25000000186264515, + "rewards/cosine_scaled_reward": 0.01158975763246417, + "step": 352 + }, + { + "clip_fraction": 0.0, + "completion_length": 2533.7916946411133, + "epoch": 0.4034285714285714, + "grad_norm": 0.07949981093406677, + "kl": 4.393863491714001e-05, + "lambda_div_used": 0.6204324588179588, + "learning_rate": 3.195807108082429e-07, + "loss": 0.0456, + "reward": -0.12379613053053617, + "reward_after_mean": -0.12379613053053617, + "reward_after_std": 0.6454745382070541, + "reward_before_mean": 0.203694608528167, + "reward_before_std": 0.5591385969892144, + "reward_change_max": 0.0, + "reward_change_mean": -0.327490733936429, + "reward_change_min": -0.5519858598709106, + "reward_change_std": 0.1984806014224887, + "reward_std": 0.6454745400696993, + "rewards/accuracy_reward": 0.2500000037252903, + "rewards/cosine_scaled_reward": -0.0463053984567523, + "step": 353 + }, + { + "clip_fraction": 0.0, + "completion_length": 1914.562515258789, + "epoch": 0.4045714285714286, + "grad_norm": 0.07853464037179947, + "kl": 2.3346394300460815e-05, + "lambda_div_used": 0.5937526449561119, + "learning_rate": 3.168878457820915e-07, + "loss": -0.0345, + "reward": 0.036306386813521385, + "reward_after_mean": 0.036306386813521385, + "reward_after_std": 0.5187021996825933, + "reward_before_mean": 0.4854283039458096, + "reward_before_std": 0.42800275422632694, + "reward_change_max": 0.0, + "reward_change_mean": -0.4491219110786915, + "reward_change_min": -0.637521255761385, + "reward_change_std": 0.252533134073019, + "reward_std": 0.5187022183090448, + "rewards/accuracy_reward": 0.33333334140479565, + "rewards/cosine_scaled_reward": 0.15209495089948177, + "step": 354 + }, + { + "clip_fraction": 0.0, + "completion_length": 2335.8958740234375, + "epoch": 0.4057142857142857, + "grad_norm": 0.09039249271154404, + "kl": 2.7535483241081238e-05, + "lambda_div_used": 0.6392721608281136, + "learning_rate": 3.142063423134644e-07, + "loss": 0.0158, + "reward": 0.017912205308675766, + "reward_after_mean": 0.017912205308675766, + "reward_after_std": 0.6705970745533705, + "reward_before_mean": 0.355070261284709, + "reward_before_std": 0.6396346259862185, + "reward_change_max": 0.0, + "reward_change_mean": -0.33715808019042015, + "reward_change_min": -0.5581717267632484, + "reward_change_std": 0.21758130192756653, + "reward_std": 0.6705970987677574, + "rewards/accuracy_reward": 0.354166679084301, + "rewards/cosine_scaled_reward": 0.0009035973343998194, + "step": 355 + }, + { + "clip_fraction": 0.0, + "completion_length": 2585.4791870117188, + "epoch": 0.40685714285714286, + "grad_norm": 0.06332956254482269, + "kl": 1.6089528799057007e-05, + "lambda_div_used": 0.6472217217087746, + "learning_rate": 3.115363310950578e-07, + "loss": -0.0329, + "reward": -0.00402236171066761, + "reward_after_mean": -0.00402236171066761, + "reward_after_std": 0.6937260664999485, + "reward_before_mean": 0.2978215580806136, + "reward_before_std": 0.6849680617451668, + "reward_change_max": 0.0, + "reward_change_mean": -0.3018439058214426, + "reward_change_min": -0.4991762898862362, + "reward_change_std": 0.2024542335420847, + "reward_std": 0.6937260907143354, + "rewards/accuracy_reward": 0.2916666753590107, + "rewards/cosine_scaled_reward": 0.00615486316382885, + "step": 356 + }, + { + "clip_fraction": 0.0, + "completion_length": 3264.8125610351562, + "epoch": 0.408, + "grad_norm": 0.057182345539331436, + "kl": 2.91336327791214e-05, + "lambda_div_used": 0.5862653851509094, + "learning_rate": 3.0887794225945143e-07, + "loss": 0.0104, + "reward": -0.31888771802186966, + "reward_after_mean": -0.31888771802186966, + "reward_after_std": 0.43322005309164524, + "reward_before_mean": -0.049007685855031013, + "reward_before_std": 0.39840223640203476, + "reward_change_max": 0.0, + "reward_change_mean": -0.2698800265789032, + "reward_change_min": -0.48054200410842896, + "reward_change_std": 0.1758509548380971, + "reward_std": 0.43322005309164524, + "rewards/accuracy_reward": 0.1041666679084301, + "rewards/cosine_scaled_reward": -0.1531743509694934, + "step": 357 + }, + { + "clip_fraction": 0.0, + "completion_length": 3050.2500534057617, + "epoch": 0.40914285714285714, + "grad_norm": 0.05330098420381546, + "kl": 2.4370267055928707e-05, + "lambda_div_used": 0.6431510671973228, + "learning_rate": 3.062313053727671e-07, + "loss": 0.003, + "reward": -0.07201657444238663, + "reward_after_mean": -0.07201657444238663, + "reward_after_std": 0.688179362565279, + "reward_before_mean": 0.2027557131368667, + "reward_before_std": 0.6673171781003475, + "reward_change_max": 0.0, + "reward_change_mean": -0.27477228827774525, + "reward_change_min": -0.4944618083536625, + "reward_change_std": 0.18934499006718397, + "reward_std": 0.6881793700158596, + "rewards/accuracy_reward": 0.2291666716337204, + "rewards/cosine_scaled_reward": -0.026410957798361778, + "step": 358 + }, + { + "clip_fraction": 0.0, + "completion_length": 2671.937530517578, + "epoch": 0.4102857142857143, + "grad_norm": 0.07592163234949112, + "kl": 3.884732723236084e-05, + "lambda_div_used": 0.5795001536607742, + "learning_rate": 3.0359654942835247e-07, + "loss": 0.0464, + "reward": -0.2641681991517544, + "reward_after_mean": -0.2641681991517544, + "reward_after_std": 0.4112328961491585, + "reward_before_mean": 0.03885669261217117, + "reward_before_std": 0.36149344593286514, + "reward_change_max": 0.0, + "reward_change_mean": -0.30302487686276436, + "reward_change_min": -0.4962599165737629, + "reward_change_std": 0.18252001702785492, + "reward_std": 0.4112329035997391, + "rewards/accuracy_reward": 0.1458333395421505, + "rewards/cosine_scaled_reward": -0.10697665251791477, + "step": 359 + }, + { + "clip_fraction": 0.0, + "completion_length": 2798.937530517578, + "epoch": 0.4114285714285714, + "grad_norm": 0.06665439158678055, + "kl": 3.1637027859687805e-05, + "lambda_div_used": 0.6062901616096497, + "learning_rate": 3.0097380284049523e-07, + "loss": -0.0029, + "reward": -0.0033570118248462677, + "reward_after_mean": -0.0033570118248462677, + "reward_after_std": 0.5593161657452583, + "reward_before_mean": 0.4031880460679531, + "reward_before_std": 0.4958424214273691, + "reward_change_max": 0.0, + "reward_change_mean": -0.40654509887099266, + "reward_change_min": -0.6473257802426815, + "reward_change_std": 0.25588439870625734, + "reward_std": 0.5593161787837744, + "rewards/accuracy_reward": 0.3333333358168602, + "rewards/cosine_scaled_reward": 0.0698547288775444, + "step": 360 + }, + { + "clip_fraction": 0.0, + "completion_length": 3150.625030517578, + "epoch": 0.4125714285714286, + "grad_norm": 0.06971946358680725, + "kl": 2.9014074243605137e-05, + "lambda_div_used": 0.6219506114721298, + "learning_rate": 2.9836319343816397e-07, + "loss": 0.0048, + "reward": -0.11981392558664083, + "reward_after_mean": -0.11981392558664083, + "reward_after_std": 0.5868565142154694, + "reward_before_mean": 0.17182117886841297, + "reward_before_std": 0.564973471686244, + "reward_change_max": 0.0, + "reward_change_mean": -0.29163510724902153, + "reward_change_min": -0.5129627995193005, + "reward_change_std": 0.19524423126131296, + "reward_std": 0.5868565402925014, + "rewards/accuracy_reward": 0.22916666977107525, + "rewards/cosine_scaled_reward": -0.057345494627952576, + "step": 361 + }, + { + "clip_fraction": 0.0, + "completion_length": 1471.916706085205, + "epoch": 0.4137142857142857, + "grad_norm": 0.12450961768627167, + "kl": 2.4568289518356323e-05, + "lambda_div_used": 0.571638435125351, + "learning_rate": 2.9576484845877793e-07, + "loss": -0.0303, + "reward": 0.013313630130141973, + "reward_after_mean": 0.013313630130141973, + "reward_after_std": 0.4897063076496124, + "reward_before_mean": 0.505017813295126, + "reward_before_std": 0.32288316125050187, + "reward_change_max": 0.0, + "reward_change_mean": -0.4917041715234518, + "reward_change_min": -0.6615025624632835, + "reward_change_std": 0.25815817900002, + "reward_std": 0.4897063188254833, + "rewards/accuracy_reward": 0.37500000558793545, + "rewards/cosine_scaled_reward": 0.1300177900120616, + "step": 362 + }, + { + "clip_fraction": 0.0, + "completion_length": 2174.270881652832, + "epoch": 0.41485714285714287, + "grad_norm": 0.08419760316610336, + "kl": 4.357472062110901e-05, + "lambda_div_used": 0.6037088930606842, + "learning_rate": 2.931788945420058e-07, + "loss": 0.0176, + "reward": -0.09517315030097961, + "reward_after_mean": -0.09517315030097961, + "reward_after_std": 0.57970448769629, + "reward_before_mean": 0.282845395617187, + "reward_before_std": 0.4830307289958, + "reward_change_max": 0.0, + "reward_change_mean": -0.37801856361329556, + "reward_change_min": -0.5875305943191051, + "reward_change_std": 0.22446841653436422, + "reward_std": 0.57970448769629, + "rewards/accuracy_reward": 0.25000000186264515, + "rewards/cosine_scaled_reward": 0.03284538397565484, + "step": 363 + }, + { + "clip_fraction": 0.0, + "completion_length": 2614.9167098999023, + "epoch": 0.416, + "grad_norm": 0.07822614908218384, + "kl": 2.289540134370327e-05, + "lambda_div_used": 0.5602610111236572, + "learning_rate": 2.9060545772359305e-07, + "loss": 0.0079, + "reward": -0.4077296704053879, + "reward_after_mean": -0.4077296704053879, + "reward_after_std": 0.3345623780041933, + "reward_before_mean": -0.1330121699720621, + "reward_before_std": 0.27498441375792027, + "reward_change_max": 0.0, + "reward_change_mean": -0.2747174873948097, + "reward_change_min": -0.4260003827512264, + "reward_change_std": 0.16245513781905174, + "reward_std": 0.3345623817294836, + "rewards/accuracy_reward": 0.0625, + "rewards/cosine_scaled_reward": -0.195512181147933, + "step": 364 + }, + { + "clip_fraction": 0.0, + "completion_length": 3003.604202270508, + "epoch": 0.41714285714285715, + "grad_norm": 0.04828161001205444, + "kl": 6.87967985868454e-06, + "lambda_div_used": 0.6036554351449013, + "learning_rate": 2.8804466342921987e-07, + "loss": 0.0555, + "reward": -0.31679424038156867, + "reward_after_mean": -0.31679424038156867, + "reward_after_std": 0.5247047282755375, + "reward_before_mean": -0.07733823172748089, + "reward_before_std": 0.4768796032294631, + "reward_change_max": 0.0, + "reward_change_mean": -0.23945600911974907, + "reward_change_min": -0.42952996119856834, + "reward_change_std": 0.15284609980881214, + "reward_std": 0.5247047450393438, + "rewards/accuracy_reward": 0.10416666977107525, + "rewards/cosine_scaled_reward": -0.18150490219704807, + "step": 365 + }, + { + "clip_fraction": 0.0, + "completion_length": 1893.00004196167, + "epoch": 0.41828571428571426, + "grad_norm": 0.08456390351057053, + "kl": 1.6473233699798584e-05, + "lambda_div_used": 0.6091032102704048, + "learning_rate": 2.854966364683872e-07, + "loss": -0.0195, + "reward": 0.005720527842640877, + "reward_after_mean": 0.005720527842640877, + "reward_after_std": 0.6514943428337574, + "reward_before_mean": 0.43899719044566154, + "reward_before_std": 0.49832448456436396, + "reward_change_max": 0.0, + "reward_change_mean": -0.4332766607403755, + "reward_change_min": -0.6058122254908085, + "reward_change_std": 0.23482245951890945, + "reward_std": 0.6514943763613701, + "rewards/accuracy_reward": 0.39583333767950535, + "rewards/cosine_scaled_reward": 0.043163834139704704, + "step": 366 + }, + { + "clip_fraction": 0.0, + "completion_length": 2780.937515258789, + "epoch": 0.41942857142857143, + "grad_norm": 0.06994574517011642, + "kl": 2.946704626083374e-05, + "lambda_div_used": 0.6079575195908546, + "learning_rate": 2.829615010283344e-07, + "loss": 0.0087, + "reward": -0.16347008850425482, + "reward_after_mean": -0.16347008850425482, + "reward_after_std": 0.5466162711381912, + "reward_before_mean": 0.1403359491378069, + "reward_before_std": 0.49341694079339504, + "reward_change_max": 0.0, + "reward_change_mean": -0.30380604416131973, + "reward_change_min": -0.4883100688457489, + "reward_change_std": 0.18378268275409937, + "reward_std": 0.5466162823140621, + "rewards/accuracy_reward": 0.1875000074505806, + "rewards/cosine_scaled_reward": -0.0471640374744311, + "step": 367 + }, + { + "clip_fraction": 0.0, + "completion_length": 2967.041679382324, + "epoch": 0.4205714285714286, + "grad_norm": 0.07414139062166214, + "kl": 4.155188798904419e-05, + "lambda_div_used": 0.5803077146410942, + "learning_rate": 2.8043938066798645e-07, + "loss": 0.0504, + "reward": -0.27402403950691223, + "reward_after_mean": -0.27402403950691223, + "reward_after_std": 0.4059589561074972, + "reward_before_mean": 0.008191026747226715, + "reward_before_std": 0.371637674048543, + "reward_change_max": 0.0, + "reward_change_mean": -0.28221505135297775, + "reward_change_min": -0.44202784821391106, + "reward_change_std": 0.17603088915348053, + "reward_std": 0.40595896542072296, + "rewards/accuracy_reward": 0.1666666716337204, + "rewards/cosine_scaled_reward": -0.15847563743591309, + "step": 368 + }, + { + "clip_fraction": 0.0, + "completion_length": 2863.1667404174805, + "epoch": 0.4217142857142857, + "grad_norm": 0.07171155512332916, + "kl": 2.0891427993774414e-05, + "lambda_div_used": 0.6472674459218979, + "learning_rate": 2.7793039831193133e-07, + "loss": -0.0044, + "reward": -0.04304695222526789, + "reward_after_mean": -0.04304695222526789, + "reward_after_std": 0.687617726624012, + "reward_before_mean": 0.25146727729588747, + "reward_before_std": 0.6888436311855912, + "reward_change_max": 0.0, + "reward_change_mean": -0.29451421834528446, + "reward_change_min": -0.6169496960937977, + "reward_change_std": 0.22693553566932678, + "reward_std": 0.6876177359372377, + "rewards/accuracy_reward": 0.2500000037252903, + "rewards/cosine_scaled_reward": 0.0014672745019197464, + "step": 369 + }, + { + "clip_fraction": 0.0, + "completion_length": 3261.2291870117188, + "epoch": 0.4228571428571429, + "grad_norm": 0.058094967156648636, + "kl": 2.1554529666900635e-05, + "lambda_div_used": 0.5657742545008659, + "learning_rate": 2.7543467624442956e-07, + "loss": 0.0031, + "reward": -0.2483972143381834, + "reward_after_mean": -0.2483972143381834, + "reward_after_std": 0.4499538466334343, + "reward_before_mean": 0.13386818021535873, + "reward_before_std": 0.29791492875665426, + "reward_change_max": 0.0, + "reward_change_mean": -0.38226539455354214, + "reward_change_min": -0.51585279032588, + "reward_change_std": 0.19617348536849022, + "reward_std": 0.4499538540840149, + "rewards/accuracy_reward": 0.1666666679084301, + "rewards/cosine_scaled_reward": -0.03279848676174879, + "step": 370 + }, + { + "clip_fraction": 0.0, + "completion_length": 1885.645866394043, + "epoch": 0.424, + "grad_norm": 0.12381042540073395, + "kl": 1.5240773791447282e-05, + "lambda_div_used": 0.5325883999466896, + "learning_rate": 2.729523361034538e-07, + "loss": -0.0304, + "reward": -0.14332129061222076, + "reward_after_mean": -0.14332129061222076, + "reward_after_std": 0.3713335506618023, + "reward_before_mean": 0.3775896169245243, + "reward_before_std": 0.14313361048698425, + "reward_change_max": 0.0, + "reward_change_mean": -0.5209108907729387, + "reward_change_min": -0.6799871250987053, + "reward_change_std": 0.26304140500724316, + "reward_std": 0.3713335543870926, + "rewards/accuracy_reward": 0.375, + "rewards/cosine_scaled_reward": 0.0025896020233631134, + "step": 371 + }, + { + "clip_fraction": 0.0, + "completion_length": 3061.6250228881836, + "epoch": 0.42514285714285716, + "grad_norm": 0.0565616711974144, + "kl": 1.3771001249551773e-05, + "lambda_div_used": 0.6153379678726196, + "learning_rate": 2.7048349887476037e-07, + "loss": 0.0019, + "reward": 0.08842738252133131, + "reward_after_mean": 0.08842738252133131, + "reward_after_std": 0.6899000462144613, + "reward_before_mean": 0.5513297475408763, + "reward_before_std": 0.5352799613028765, + "reward_change_max": 0.0, + "reward_change_mean": -0.46290238574147224, + "reward_change_min": -0.6905807815492153, + "reward_change_std": 0.26435263454914093, + "reward_std": 0.6899000480771065, + "rewards/accuracy_reward": 0.3958333358168602, + "rewards/cosine_scaled_reward": 0.15549640776589513, + "step": 372 + }, + { + "clip_fraction": 0.0, + "completion_length": 1999.791690826416, + "epoch": 0.42628571428571427, + "grad_norm": 0.11289634555578232, + "kl": 2.6244670152664185e-05, + "lambda_div_used": 0.5901408270001411, + "learning_rate": 2.6802828488599294e-07, + "loss": -0.0028, + "reward": -0.25084885116666555, + "reward_after_mean": -0.25084885116666555, + "reward_after_std": 0.47954990342259407, + "reward_before_mean": 0.03629123326390982, + "reward_before_std": 0.41134614683687687, + "reward_change_max": 0.0, + "reward_change_mean": -0.28714008443057537, + "reward_change_min": -0.4200817756354809, + "reward_change_std": 0.1626526527106762, + "reward_std": 0.47954992204904556, + "rewards/accuracy_reward": 0.1250000037252903, + "rewards/cosine_scaled_reward": -0.08870877418667078, + "step": 373 + }, + { + "clip_fraction": 0.0, + "completion_length": 2700.791717529297, + "epoch": 0.42742857142857144, + "grad_norm": 0.0688440129160881, + "kl": 2.674013376235962e-05, + "lambda_div_used": 0.6036395579576492, + "learning_rate": 2.655868138008171e-07, + "loss": -0.0525, + "reward": -0.17511339485645294, + "reward_after_mean": -0.17511339485645294, + "reward_after_std": 0.5073369853198528, + "reward_before_mean": 0.13348117470741272, + "reward_before_std": 0.48259105905890465, + "reward_change_max": 0.0, + "reward_change_mean": -0.30859458073973656, + "reward_change_min": -0.5309739634394646, + "reward_change_std": 0.20630291104316711, + "reward_std": 0.5073370076715946, + "rewards/accuracy_reward": 0.2083333395421505, + "rewards/cosine_scaled_reward": -0.07485215552151203, + "step": 374 + }, + { + "clip_fraction": 0.0, + "completion_length": 3033.791702270508, + "epoch": 0.42857142857142855, + "grad_norm": 0.05866050720214844, + "kl": 1.8252991139888763e-05, + "lambda_div_used": 0.6017400398850441, + "learning_rate": 2.631592046130896e-07, + "loss": 0.0477, + "reward": -0.18571221362799406, + "reward_after_mean": -0.18571221362799406, + "reward_after_std": 0.5023255608975887, + "reward_before_mean": 0.11062880232930183, + "reward_before_std": 0.46861019916832447, + "reward_change_max": 0.0, + "reward_change_mean": -0.2963410019874573, + "reward_change_min": -0.45132842287421227, + "reward_change_std": 0.1817708509042859, + "reward_std": 0.5023255832493305, + "rewards/accuracy_reward": 0.20833334140479565, + "rewards/cosine_scaled_reward": -0.097704553976655, + "step": 375 + }, + { + "clip_fraction": 0.0, + "completion_length": 2246.770851135254, + "epoch": 0.4297142857142857, + "grad_norm": 0.08337263762950897, + "kl": 8.605420589447021e-06, + "lambda_div_used": 0.5853807479143143, + "learning_rate": 2.6074557564105724e-07, + "loss": 0.0128, + "reward": -0.3334618601948023, + "reward_after_mean": -0.3334618601948023, + "reward_after_std": 0.4486316703259945, + "reward_before_mean": -0.05933803477091715, + "reward_before_std": 0.3892780668102205, + "reward_change_max": 0.0, + "reward_change_mean": -0.2741238009184599, + "reward_change_min": -0.46821001917123795, + "reward_change_std": 0.1663337228819728, + "reward_std": 0.44863167591392994, + "rewards/accuracy_reward": 0.10416666977107525, + "rewards/cosine_scaled_reward": -0.163504708558321, + "step": 376 + }, + { + "clip_fraction": 0.0, + "completion_length": 3488.3541870117188, + "epoch": 0.4308571428571429, + "grad_norm": 0.055966660380363464, + "kl": 2.5756657123565674e-05, + "lambda_div_used": 0.5742340236902237, + "learning_rate": 2.583460445215911e-07, + "loss": -0.0032, + "reward": -0.3738445993512869, + "reward_after_mean": -0.3738445993512869, + "reward_after_std": 0.39967909548431635, + "reward_before_mean": -0.10715527582215145, + "reward_before_std": 0.33878533728420734, + "reward_change_max": 0.0, + "reward_change_mean": -0.26668932288885117, + "reward_change_min": -0.4109174869954586, + "reward_change_std": 0.15686567965894938, + "reward_std": 0.39967911317944527, + "rewards/accuracy_reward": 0.08333333395421505, + "rewards/cosine_scaled_reward": -0.19048861227929592, + "step": 377 + }, + { + "clip_fraction": 0.0, + "completion_length": 2134.3333740234375, + "epoch": 0.432, + "grad_norm": 0.13181807100772858, + "kl": 2.8654932975769043e-05, + "lambda_div_used": 0.6004306748509407, + "learning_rate": 2.5596072820445254e-07, + "loss": -0.0331, + "reward": -0.06989812850952148, + "reward_after_mean": -0.06989812850952148, + "reward_after_std": 0.5459046512842178, + "reward_before_mean": 0.31881395215168595, + "reward_before_std": 0.46263560838997364, + "reward_change_max": 0.0, + "reward_change_mean": -0.3887120857834816, + "reward_change_min": -0.5917952992022038, + "reward_change_std": 0.23005317710340023, + "reward_std": 0.5459046605974436, + "rewards/accuracy_reward": 0.29166666977107525, + "rewards/cosine_scaled_reward": 0.027147281914949417, + "step": 378 + }, + { + "clip_fraction": 0.0, + "completion_length": 3121.0416870117188, + "epoch": 0.43314285714285716, + "grad_norm": 0.07427296042442322, + "kl": 1.5733763575553894e-05, + "lambda_div_used": 0.5833418369293213, + "learning_rate": 2.5358974294659373e-07, + "loss": -0.0048, + "reward": -0.29101302847266197, + "reward_after_mean": -0.29101302847266197, + "reward_after_std": 0.4290795102715492, + "reward_before_mean": -0.009382706135511398, + "reward_before_std": 0.38597164303064346, + "reward_change_max": 0.0, + "reward_change_mean": -0.2816303465515375, + "reward_change_min": -0.47839244455099106, + "reward_change_std": 0.17911072820425034, + "reward_std": 0.4290795363485813, + "rewards/accuracy_reward": 0.125, + "rewards/cosine_scaled_reward": -0.13438269309699535, + "step": 379 + }, + { + "clip_fraction": 0.0, + "completion_length": 2823.416702270508, + "epoch": 0.4342857142857143, + "grad_norm": 0.06611248105764389, + "kl": 2.2601336240768433e-05, + "lambda_div_used": 0.6066281795501709, + "learning_rate": 2.512332043064913e-07, + "loss": 0.0533, + "reward": -0.17613293696194887, + "reward_after_mean": -0.17613293696194887, + "reward_after_std": 0.495351817458868, + "reward_before_mean": 0.1346770692616701, + "reward_before_std": 0.48867712169885635, + "reward_change_max": 0.0, + "reward_change_mean": -0.31081000342965126, + "reward_change_min": -0.5689719989895821, + "reward_change_std": 0.21053248085081577, + "reward_std": 0.4953518286347389, + "rewards/accuracy_reward": 0.1875000074505806, + "rewards/cosine_scaled_reward": -0.05282294252538122, + "step": 380 + }, + { + "clip_fraction": 0.0, + "completion_length": 2983.7916984558105, + "epoch": 0.43542857142857144, + "grad_norm": 0.08519254624843597, + "kl": 3.2504089176654816e-05, + "lambda_div_used": 0.6109358817338943, + "learning_rate": 2.488912271385139e-07, + "loss": 0.0165, + "reward": -0.2218069350346923, + "reward_after_mean": -0.2218069350346923, + "reward_after_std": 0.5642109382897615, + "reward_before_mean": 0.040106164291501045, + "reward_before_std": 0.5099438140168786, + "reward_change_max": 0.0, + "reward_change_mean": -0.261913089081645, + "reward_change_min": -0.45296039059758186, + "reward_change_std": 0.1642473293468356, + "reward_std": 0.5642109606415033, + "rewards/accuracy_reward": 0.1458333358168602, + "rewards/cosine_scaled_reward": -0.10572716826573014, + "step": 381 + }, + { + "clip_fraction": 0.0, + "completion_length": 2500.437545776367, + "epoch": 0.43657142857142855, + "grad_norm": 0.07325682789087296, + "kl": 3.65767627954483e-05, + "lambda_div_used": 0.5600408837199211, + "learning_rate": 2.465639255873246e-07, + "loss": -0.0096, + "reward": -0.4078812226653099, + "reward_after_mean": -0.4078812226653099, + "reward_after_std": 0.3340430334210396, + "reward_before_mean": -0.13890772312879562, + "reward_before_std": 0.27443209011107683, + "reward_change_max": 0.0, + "reward_change_mean": -0.26897350139915943, + "reward_change_min": -0.42554087564349174, + "reward_change_std": 0.15754834469407797, + "reward_std": 0.3340430427342653, + "rewards/accuracy_reward": 0.0625, + "rewards/cosine_scaled_reward": -0.20140772499144077, + "step": 382 + }, + { + "clip_fraction": 0.0, + "completion_length": 2824.291717529297, + "epoch": 0.4377142857142857, + "grad_norm": 0.08796168118715286, + "kl": 4.8138201236724854e-05, + "lambda_div_used": 0.5781937688589096, + "learning_rate": 2.4425141308231765e-07, + "loss": -0.0086, + "reward": -0.16291768848896027, + "reward_after_mean": -0.16291768848896027, + "reward_after_std": 0.387674568220973, + "reward_before_mean": 0.1924248207360506, + "reward_before_std": 0.35597400926053524, + "reward_change_max": 0.0, + "reward_change_mean": -0.3553424943238497, + "reward_change_min": -0.5255323797464371, + "reward_change_std": 0.21135967783629894, + "reward_std": 0.3876745719462633, + "rewards/accuracy_reward": 0.2083333432674408, + "rewards/cosine_scaled_reward": -0.01590852066874504, + "step": 383 + }, + { + "clip_fraction": 0.0, + "completion_length": 2347.6042404174805, + "epoch": 0.43885714285714283, + "grad_norm": 0.09212023764848709, + "kl": 2.508983016014099e-05, + "lambda_div_used": 0.6615950018167496, + "learning_rate": 2.4195380233209006e-07, + "loss": 0.0408, + "reward": 0.27819300815463066, + "reward_after_mean": 0.27819300815463066, + "reward_after_std": 0.7268820777535439, + "reward_before_mean": 0.6785084716975689, + "reward_before_std": 0.7540020374581218, + "reward_change_max": 0.0, + "reward_change_mean": -0.4003154691308737, + "reward_change_min": -0.6902611702680588, + "reward_change_std": 0.283959056250751, + "reward_std": 0.7268821019679308, + "rewards/accuracy_reward": 0.4583333432674408, + "rewards/cosine_scaled_reward": 0.22017512656748295, + "step": 384 + }, + { + "clip_fraction": 0.0, + "completion_length": 2711.958396911621, + "epoch": 0.44, + "grad_norm": 0.07038652151823044, + "kl": 1.942366361618042e-05, + "lambda_div_used": 0.6165113672614098, + "learning_rate": 2.3967120531894857e-07, + "loss": 0.012, + "reward": -0.25626325886696577, + "reward_after_mean": -0.25626325886696577, + "reward_after_std": 0.5892766248434782, + "reward_before_mean": -0.0074533987790346146, + "reward_before_std": 0.5360177559778094, + "reward_change_max": 0.0, + "reward_change_mean": -0.24880987033247948, + "reward_change_min": -0.44854912906885147, + "reward_change_std": 0.1595793990418315, + "reward_std": 0.5892766322940588, + "rewards/accuracy_reward": 0.1250000037252903, + "rewards/cosine_scaled_reward": -0.13245339877903461, + "step": 385 + }, + { + "clip_fraction": 0.0, + "completion_length": 2889.1042404174805, + "epoch": 0.44114285714285717, + "grad_norm": 0.060970280319452286, + "kl": 1.8231570720672607e-05, + "lambda_div_used": 0.5940364003181458, + "learning_rate": 2.374037332934512e-07, + "loss": -0.0287, + "reward": -0.24576714914292097, + "reward_after_mean": -0.24576714914292097, + "reward_after_std": 0.4883785657584667, + "reward_before_mean": 0.04634229093790054, + "reward_before_std": 0.4279829766601324, + "reward_change_max": 0.0, + "reward_change_mean": -0.29210945032536983, + "reward_change_min": -0.4432514049112797, + "reward_change_std": 0.17009661067277193, + "reward_std": 0.48837856762111187, + "rewards/accuracy_reward": 0.1250000037252903, + "rewards/cosine_scaled_reward": -0.07865770626813173, + "step": 386 + }, + { + "clip_fraction": 0.0, + "completion_length": 2997.3125, + "epoch": 0.4422857142857143, + "grad_norm": 0.07781542837619781, + "kl": 1.3821758329868317e-05, + "lambda_div_used": 0.5551474094390869, + "learning_rate": 2.3515149676898552e-07, + "loss": -0.0529, + "reward": -0.38486793637275696, + "reward_after_mean": -0.38486793637275696, + "reward_after_std": 0.31045267172157764, + "reward_before_mean": -0.10442159324884415, + "reward_before_std": 0.24840335873886943, + "reward_change_max": 0.0, + "reward_change_mean": -0.28044634498655796, + "reward_change_min": -0.4241391494870186, + "reward_change_std": 0.15805031638592482, + "reward_std": 0.31045267917215824, + "rewards/accuracy_reward": 0.1041666716337204, + "rewards/cosine_scaled_reward": -0.20858826488256454, + "step": 387 + }, + { + "clip_fraction": 0.0, + "completion_length": 2572.2708587646484, + "epoch": 0.44342857142857145, + "grad_norm": 0.06563723832368851, + "kl": 2.9848888516426086e-06, + "lambda_div_used": 0.5865650475025177, + "learning_rate": 2.3291460551638237e-07, + "loss": 0.0407, + "reward": -0.15434425324201584, + "reward_after_mean": -0.15434425324201584, + "reward_after_std": 0.47601727209985256, + "reward_before_mean": 0.20021704956889153, + "reward_before_std": 0.39916726760566235, + "reward_change_max": 0.0, + "reward_change_mean": -0.354561323300004, + "reward_change_min": -0.5467559210956097, + "reward_change_std": 0.21171134896576405, + "reward_std": 0.47601727209985256, + "rewards/accuracy_reward": 0.2708333358168602, + "rewards/cosine_scaled_reward": -0.07061627879738808, + "step": 388 + }, + { + "clip_fraction": 0.0, + "completion_length": 2650.937545776367, + "epoch": 0.44457142857142856, + "grad_norm": 0.07840488851070404, + "kl": 1.73286534845829e-05, + "lambda_div_used": 0.6045578718185425, + "learning_rate": 2.306931685585657e-07, + "loss": -0.0394, + "reward": -0.26122746989130974, + "reward_after_mean": -0.26122746989130974, + "reward_after_std": 0.5210577324032784, + "reward_before_mean": 0.002998221665620804, + "reward_before_std": 0.486898148432374, + "reward_change_max": 0.0, + "reward_change_mean": -0.26422569528222084, + "reward_change_min": -0.4951511509716511, + "reward_change_std": 0.18348107766360044, + "reward_std": 0.5210577566176653, + "rewards/accuracy_reward": 0.1458333358168602, + "rewards/cosine_scaled_reward": -0.1428351059439592, + "step": 389 + }, + { + "clip_fraction": 0.0, + "completion_length": 3007.666679382324, + "epoch": 0.44571428571428573, + "grad_norm": 0.07837636768817902, + "kl": 1.6082078218460083e-05, + "lambda_div_used": 0.61600511521101, + "learning_rate": 2.2848729416523859e-07, + "loss": 0.0713, + "reward": -0.18647840432822704, + "reward_after_mean": -0.18647840432822704, + "reward_after_std": 0.5758322961628437, + "reward_before_mean": 0.08954079262912273, + "reward_before_std": 0.5347468825057149, + "reward_change_max": 0.0, + "reward_change_mean": -0.27601918019354343, + "reward_change_min": -0.47189946845173836, + "reward_change_std": 0.17452176753431559, + "reward_std": 0.5758323054760695, + "rewards/accuracy_reward": 0.1666666716337204, + "rewards/cosine_scaled_reward": -0.07712590089067817, + "step": 390 + }, + { + "clip_fraction": 0.0, + "completion_length": 2716.708351135254, + "epoch": 0.44685714285714284, + "grad_norm": 0.09803462773561478, + "kl": 2.5559216737747192e-05, + "lambda_div_used": 0.6362191960215569, + "learning_rate": 2.2629708984760706e-07, + "loss": 0.0166, + "reward": -0.013867436617147177, + "reward_after_mean": -0.013867436617147177, + "reward_after_std": 0.6719868443906307, + "reward_before_mean": 0.30256712157279253, + "reward_before_std": 0.6324740117415786, + "reward_change_max": 0.0, + "reward_change_mean": -0.3164345696568489, + "reward_change_min": -0.5261501334607601, + "reward_change_std": 0.20623697619885206, + "reward_std": 0.6719868592917919, + "rewards/accuracy_reward": 0.2708333432674408, + "rewards/cosine_scaled_reward": 0.03173377411440015, + "step": 391 + }, + { + "clip_fraction": 0.0, + "completion_length": 2130.9791984558105, + "epoch": 0.448, + "grad_norm": 0.11540649086236954, + "kl": 1.8610619008541107e-05, + "lambda_div_used": 0.6027960926294327, + "learning_rate": 2.2412266235313973e-07, + "loss": -0.0919, + "reward": -0.16835473664104939, + "reward_after_mean": -0.16835473664104939, + "reward_after_std": 0.5122340489178896, + "reward_before_mean": 0.1393747702240944, + "reward_before_std": 0.4794600326567888, + "reward_change_max": 0.0, + "reward_change_mean": -0.3077295087277889, + "reward_change_min": -0.4898943528532982, + "reward_change_std": 0.1954572731629014, + "reward_std": 0.5122340768575668, + "rewards/accuracy_reward": 0.16666666977107525, + "rewards/cosine_scaled_reward": -0.027291906299069524, + "step": 392 + }, + { + "clip_fraction": 0.0, + "completion_length": 2649.562530517578, + "epoch": 0.4491428571428571, + "grad_norm": 0.07226411253213882, + "kl": 2.648681402206421e-05, + "lambda_div_used": 0.5953627824783325, + "learning_rate": 2.2196411766036487e-07, + "loss": -0.067, + "reward": -0.20140517689287663, + "reward_after_mean": -0.20140517689287663, + "reward_after_std": 0.4766168761998415, + "reward_before_mean": 0.09705937840044498, + "reward_before_std": 0.4375506564974785, + "reward_change_max": 0.0, + "reward_change_mean": -0.2984645199030638, + "reward_change_min": -0.47758448868989944, + "reward_change_std": 0.18242334388196468, + "reward_std": 0.47661688551306725, + "rewards/accuracy_reward": 0.18750000558793545, + "rewards/cosine_scaled_reward": -0.09044064581394196, + "step": 393 + }, + { + "clip_fraction": 0.0, + "completion_length": 3104.1458740234375, + "epoch": 0.4502857142857143, + "grad_norm": 0.07446404546499252, + "kl": 2.555176615715027e-05, + "lambda_div_used": 0.5502220839262009, + "learning_rate": 2.1982156097370557e-07, + "loss": -0.1089, + "reward": -0.46874738670885563, + "reward_after_mean": -0.46874738670885563, + "reward_after_std": 0.30342659167945385, + "reward_before_mean": -0.20392105542123318, + "reward_before_std": 0.22309745661914349, + "reward_change_max": 0.0, + "reward_change_mean": -0.2648263294249773, + "reward_change_min": -0.3818593733012676, + "reward_change_std": 0.14172559697180986, + "reward_std": 0.30342659167945385, + "rewards/accuracy_reward": 0.02083333395421505, + "rewards/cosine_scaled_reward": -0.22475438751280308, + "step": 394 + }, + { + "clip_fraction": 0.0, + "completion_length": 2284.6458435058594, + "epoch": 0.4514285714285714, + "grad_norm": 0.12902304530143738, + "kl": 4.9501657485961914e-05, + "lambda_div_used": 0.5602920204401016, + "learning_rate": 2.1769509671835223e-07, + "loss": -0.1675, + "reward": -0.3165151756256819, + "reward_after_mean": -0.3165151756256819, + "reward_after_std": 0.3793674483895302, + "reward_before_mean": 0.0216768067330122, + "reward_before_std": 0.27139727398753166, + "reward_change_max": 0.0, + "reward_change_mean": -0.3381919823586941, + "reward_change_min": -0.48955149203538895, + "reward_change_std": 0.17990652937442064, + "reward_std": 0.3793674521148205, + "rewards/accuracy_reward": 0.14583333395421505, + "rewards/cosine_scaled_reward": -0.12415653187781572, + "step": 395 + }, + { + "clip_fraction": 0.0, + "completion_length": 3048.1458587646484, + "epoch": 0.45257142857142857, + "grad_norm": 0.054839495569467545, + "kl": 7.04331323504448e-05, + "lambda_div_used": 0.6417915895581245, + "learning_rate": 2.1558482853517253e-07, + "loss": 0.0192, + "reward": -0.09411877207458019, + "reward_after_mean": -0.09411877207458019, + "reward_after_std": 0.6742421016097069, + "reward_before_mean": 0.1871709941624431, + "reward_before_std": 0.659186695702374, + "reward_change_max": 0.0, + "reward_change_mean": -0.2812897562980652, + "reward_change_min": -0.5664796940982342, + "reward_change_std": 0.20405743923038244, + "reward_std": 0.6742421071976423, + "rewards/accuracy_reward": 0.20833333767950535, + "rewards/cosine_scaled_reward": -0.021162351593375206, + "step": 396 + }, + { + "clip_fraction": 0.0, + "completion_length": 2982.750030517578, + "epoch": 0.45371428571428574, + "grad_norm": 0.05798732861876488, + "kl": 2.0850449800491333e-05, + "lambda_div_used": 0.5765915215015411, + "learning_rate": 2.134908592756607e-07, + "loss": 0.0391, + "reward": -0.36006474308669567, + "reward_after_mean": -0.36006474308669567, + "reward_after_std": 0.40458301082253456, + "reward_before_mean": -0.08499279711395502, + "reward_before_std": 0.3492864612489939, + "reward_change_max": 0.0, + "reward_change_mean": -0.2750719413161278, + "reward_change_min": -0.4323217496275902, + "reward_change_std": 0.16262990981340408, + "reward_std": 0.40458302199840546, + "rewards/accuracy_reward": 0.08333333395421505, + "rewards/cosine_scaled_reward": -0.16832613572478294, + "step": 397 + }, + { + "clip_fraction": 0.0, + "completion_length": 2822.5209045410156, + "epoch": 0.45485714285714285, + "grad_norm": 0.06360434740781784, + "kl": 2.2810418158769608e-05, + "lambda_div_used": 0.6480221897363663, + "learning_rate": 2.1141329099692406e-07, + "loss": 0.0465, + "reward": -0.09229415841400623, + "reward_after_mean": -0.09229415841400623, + "reward_after_std": 0.7079674638807774, + "reward_before_mean": 0.1752314588520676, + "reward_before_std": 0.6917341919615865, + "reward_change_max": 0.0, + "reward_change_mean": -0.26752561889588833, + "reward_change_min": -0.5181693434715271, + "reward_change_std": 0.19102454278618097, + "reward_std": 0.7079674787819386, + "rewards/accuracy_reward": 0.2083333358168602, + "rewards/cosine_scaled_reward": -0.03310186788439751, + "step": 398 + }, + { + "clip_fraction": 0.0, + "completion_length": 2191.583351135254, + "epoch": 0.456, + "grad_norm": 0.07585066556930542, + "kl": 3.449246287345886e-05, + "lambda_div_used": 0.6071069464087486, + "learning_rate": 2.0935222495670968e-07, + "loss": -0.0428, + "reward": -0.06968086212873459, + "reward_after_mean": -0.06968086212873459, + "reward_after_std": 0.5623702071607113, + "reward_before_mean": 0.29000685061328113, + "reward_before_std": 0.4939764440059662, + "reward_change_max": 0.0, + "reward_change_mean": -0.35968772508203983, + "reward_change_min": -0.5781176537275314, + "reward_change_std": 0.2176226656883955, + "reward_std": 0.5623702295124531, + "rewards/accuracy_reward": 0.27083333767950535, + "rewards/cosine_scaled_reward": 0.01917351223528385, + "step": 399 + }, + { + "clip_fraction": 0.0, + "completion_length": 1814.4375305175781, + "epoch": 0.45714285714285713, + "grad_norm": 0.09906096011400223, + "kl": 3.718771040439606e-05, + "lambda_div_used": 0.615766242146492, + "learning_rate": 2.0730776160846853e-07, + "loss": -0.0203, + "reward": 0.023047026246786118, + "reward_after_mean": 0.023047026246786118, + "reward_after_std": 0.6236102003604174, + "reward_before_mean": 0.43256790889427066, + "reward_before_std": 0.5322064086794853, + "reward_change_max": 0.0, + "reward_change_mean": -0.40952087566256523, + "reward_change_min": -0.6220400035381317, + "reward_change_std": 0.2354581467807293, + "reward_std": 0.62361023388803, + "rewards/accuracy_reward": 0.3125000074505806, + "rewards/cosine_scaled_reward": 0.12006791215389967, + "step": 400 + }, + { + "clip_fraction": 0.0, + "completion_length": 3091.7916870117188, + "epoch": 0.4582857142857143, + "grad_norm": 0.05475914105772972, + "kl": 3.5446137189865112e-06, + "lambda_div_used": 0.562153548002243, + "learning_rate": 2.0528000059645995e-07, + "loss": 0.0508, + "reward": -0.29909564182162285, + "reward_after_mean": -0.29909564182162285, + "reward_after_std": 0.3837758805602789, + "reward_before_mean": 0.04575100168585777, + "reward_before_std": 0.28164876997470856, + "reward_change_max": 0.0, + "reward_change_mean": -0.3448466807603836, + "reward_change_min": -0.5300594680011272, + "reward_change_std": 0.19381076097488403, + "reward_std": 0.38377588987350464, + "rewards/accuracy_reward": 0.1666666679084301, + "rewards/cosine_scaled_reward": -0.12091565364971757, + "step": 401 + }, + { + "clip_fraction": 0.0, + "completion_length": 2440.1458740234375, + "epoch": 0.4594285714285714, + "grad_norm": 0.11770489811897278, + "kl": 1.8846243619918823e-05, + "lambda_div_used": 0.5605455562472343, + "learning_rate": 2.032690407508949e-07, + "loss": -0.0261, + "reward": -0.2383374497294426, + "reward_after_mean": -0.2383374497294426, + "reward_after_std": 0.3794025480747223, + "reward_before_mean": 0.14294641837477684, + "reward_before_std": 0.27562023140490055, + "reward_change_max": 0.0, + "reward_change_mean": -0.38128384202718735, + "reward_change_min": -0.5627436973154545, + "reward_change_std": 0.2136234436184168, + "reward_std": 0.3794025518000126, + "rewards/accuracy_reward": 0.1875, + "rewards/cosine_scaled_reward": -0.044553620740771294, + "step": 402 + }, + { + "clip_fraction": 0.0, + "completion_length": 1985.6875343322754, + "epoch": 0.4605714285714286, + "grad_norm": 0.10363903641700745, + "kl": 2.3838132619857788e-05, + "lambda_div_used": 0.5793934315443039, + "learning_rate": 2.0127498008311922e-07, + "loss": 0.003, + "reward": -0.17537187691777945, + "reward_after_mean": -0.17537187691777945, + "reward_after_std": 0.46652381494641304, + "reward_before_mean": 0.20323466695845127, + "reward_before_std": 0.3647587588056922, + "reward_change_max": 0.0, + "reward_change_mean": -0.37860656157135963, + "reward_change_min": -0.5506669841706753, + "reward_change_std": 0.21334033645689487, + "reward_std": 0.4665238317102194, + "rewards/accuracy_reward": 0.22916666977107525, + "rewards/cosine_scaled_reward": -0.025931986048817635, + "step": 403 + }, + { + "clip_fraction": 0.0, + "completion_length": 2137.375, + "epoch": 0.4617142857142857, + "grad_norm": 0.10717064887285233, + "kl": 3.7202611565589905e-05, + "lambda_div_used": 0.5993036478757858, + "learning_rate": 1.9929791578083655e-07, + "loss": -0.0043, + "reward": -0.053446926176548004, + "reward_after_mean": -0.053446926176548004, + "reward_after_std": 0.4612566214054823, + "reward_before_mean": 0.3198554217815399, + "reward_before_std": 0.4542539082467556, + "reward_change_max": 0.0, + "reward_change_mean": -0.3733023554086685, + "reward_change_min": -0.5774630047380924, + "reward_change_std": 0.2339334823191166, + "reward_std": 0.4612566400319338, + "rewards/accuracy_reward": 0.2708333432674408, + "rewards/cosine_scaled_reward": 0.04902207478880882, + "step": 404 + }, + { + "clip_fraction": 0.0, + "completion_length": 2319.5000228881836, + "epoch": 0.46285714285714286, + "grad_norm": 0.13891565799713135, + "kl": 2.9403716325759888e-05, + "lambda_div_used": 0.6237343773245811, + "learning_rate": 1.9733794420337213e-07, + "loss": -0.0602, + "reward": 0.1285000964999199, + "reward_after_mean": 0.1285000964999199, + "reward_after_std": 0.7067493386566639, + "reward_before_mean": 0.5776062086224556, + "reward_before_std": 0.5746421907097101, + "reward_change_max": 0.0, + "reward_change_mean": -0.449106115847826, + "reward_change_min": -0.6593505516648293, + "reward_change_std": 0.263070298358798, + "reward_std": 0.7067493461072445, + "rewards/accuracy_reward": 0.4166666679084301, + "rewards/cosine_scaled_reward": 0.1609395444393158, + "step": 405 + }, + { + "clip_fraction": 0.0, + "completion_length": 2442.1458892822266, + "epoch": 0.464, + "grad_norm": 0.07689624279737473, + "kl": 8.527189493179321e-06, + "lambda_div_used": 0.6336105018854141, + "learning_rate": 1.9539516087697517e-07, + "loss": -0.0, + "reward": -0.11170937749557197, + "reward_after_mean": -0.11170937749557197, + "reward_after_std": 0.6453428398817778, + "reward_before_mean": 0.17541324836201966, + "reward_before_std": 0.6142212487757206, + "reward_change_max": 0.0, + "reward_change_mean": -0.2871226370334625, + "reward_change_min": -0.5107162520289421, + "reward_change_std": 0.19029081240296364, + "reward_std": 0.645342854782939, + "rewards/accuracy_reward": 0.2083333395421505, + "rewards/cosine_scaled_reward": -0.032920089550316334, + "step": 406 + }, + { + "clip_fraction": 0.0, + "completion_length": 2516.6041946411133, + "epoch": 0.46514285714285714, + "grad_norm": 0.09361904859542847, + "kl": 2.995133399963379e-05, + "lambda_div_used": 0.5926980003714561, + "learning_rate": 1.934696604901642e-07, + "loss": 0.011, + "reward": -0.16482560662552714, + "reward_after_mean": -0.16482560662552714, + "reward_after_std": 0.5236028637737036, + "reward_before_mean": 0.18611273169517517, + "reward_before_std": 0.4258856289088726, + "reward_change_max": 0.0, + "reward_change_mean": -0.35093834809958935, + "reward_change_min": -0.5517234578728676, + "reward_change_std": 0.20658957865089178, + "reward_std": 0.5236028656363487, + "rewards/accuracy_reward": 0.25000000186264515, + "rewards/cosine_scaled_reward": -0.06388726737350225, + "step": 407 + }, + { + "clip_fraction": 0.0, + "completion_length": 2709.4791984558105, + "epoch": 0.4662857142857143, + "grad_norm": 0.09306048601865768, + "kl": 3.203703090548515e-05, + "lambda_div_used": 0.5583930537104607, + "learning_rate": 1.915615368891117e-07, + "loss": -0.111, + "reward": -0.17684245621785522, + "reward_after_mean": -0.17684245621785522, + "reward_after_std": 0.42921141162514687, + "reward_before_mean": 0.2615569829940796, + "reward_before_std": 0.26226984336972237, + "reward_change_max": 0.0, + "reward_change_mean": -0.43839943781495094, + "reward_change_min": -0.6072936318814754, + "reward_change_std": 0.22694199439138174, + "reward_std": 0.42921141907572746, + "rewards/accuracy_reward": 0.27083333395421505, + "rewards/cosine_scaled_reward": -0.009276359807699919, + "step": 408 + }, + { + "clip_fraction": 0.0, + "completion_length": 3529.3541870117188, + "epoch": 0.4674285714285714, + "grad_norm": 0.048929836601018906, + "kl": 7.6089054346084595e-06, + "lambda_div_used": 0.6116252392530441, + "learning_rate": 1.8967088307307e-07, + "loss": 0.001, + "reward": -0.17062923312187195, + "reward_after_mean": -0.17062923312187195, + "reward_after_std": 0.5682655684649944, + "reward_before_mean": 0.12847105879336596, + "reward_before_std": 0.5097848381847143, + "reward_change_max": 0.0, + "reward_change_mean": -0.2991002984344959, + "reward_change_min": -0.43997345492243767, + "reward_change_std": 0.17078326642513275, + "reward_std": 0.5682655889540911, + "rewards/accuracy_reward": 0.16666667349636555, + "rewards/cosine_scaled_reward": -0.038195611676201224, + "step": 409 + }, + { + "clip_fraction": 0.0, + "completion_length": 2435.625045776367, + "epoch": 0.4685714285714286, + "grad_norm": 0.08304905891418457, + "kl": 1.5234574675559998e-05, + "lambda_div_used": 0.6268336623907089, + "learning_rate": 1.8779779118983867e-07, + "loss": 0.0499, + "reward": -0.03433734131976962, + "reward_after_mean": -0.03433734131976962, + "reward_after_std": 0.6416803412139416, + "reward_before_mean": 0.2960522407665849, + "reward_before_std": 0.5826444877311587, + "reward_change_max": 0.0, + "reward_change_mean": -0.3303895927965641, + "reward_change_min": -0.4872638136148453, + "reward_change_std": 0.19508948456496, + "reward_std": 0.6416803747415543, + "rewards/accuracy_reward": 0.25000000931322575, + "rewards/cosine_scaled_reward": 0.04605222793179564, + "step": 410 + }, + { + "clip_fraction": 0.0, + "completion_length": 3163.1041717529297, + "epoch": 0.4697142857142857, + "grad_norm": 0.06846357136964798, + "kl": 1.529604196548462e-05, + "lambda_div_used": 0.5874563306570053, + "learning_rate": 1.8594235253127372e-07, + "loss": 0.0214, + "reward": -0.22584301605820656, + "reward_after_mean": -0.22584301605820656, + "reward_after_std": 0.42311797849833965, + "reward_before_mean": 0.0774743240326643, + "reward_before_std": 0.4020446836948395, + "reward_change_max": 0.0, + "reward_change_mean": -0.3033173345029354, + "reward_change_min": -0.48705917969346046, + "reward_change_std": 0.1920422399416566, + "reward_std": 0.42311798594892025, + "rewards/accuracy_reward": 0.1666666716337204, + "rewards/cosine_scaled_reward": -0.08919236063957214, + "step": 411 + }, + { + "clip_fraction": 0.0, + "completion_length": 2888.125015258789, + "epoch": 0.47085714285714286, + "grad_norm": 0.06694348156452179, + "kl": 2.148747444152832e-05, + "lambda_div_used": 0.6005539819598198, + "learning_rate": 1.8410465752883758e-07, + "loss": 0.049, + "reward": -0.21726901549845934, + "reward_after_mean": -0.21726901549845934, + "reward_after_std": 0.48868509009480476, + "reward_before_mean": 0.08159955404698849, + "reward_before_std": 0.463420107960701, + "reward_change_max": 0.0, + "reward_change_mean": -0.2988685816526413, + "reward_change_min": -0.554147582501173, + "reward_change_std": 0.19932363275438547, + "reward_std": 0.48868510872125626, + "rewards/accuracy_reward": 0.1458333358168602, + "rewards/cosine_scaled_reward": -0.06423377990722656, + "step": 412 + }, + { + "clip_fraction": 0.0, + "completion_length": 2530.2708702087402, + "epoch": 0.472, + "grad_norm": 0.08418448269367218, + "kl": 2.208724617958069e-05, + "lambda_div_used": 0.6184473037719727, + "learning_rate": 1.822847957491922e-07, + "loss": 0.0462, + "reward": 0.05549921467900276, + "reward_after_mean": 0.05549921467900276, + "reward_after_std": 0.6010984163731337, + "reward_before_mean": 0.4671566132456064, + "reward_before_std": 0.546179112046957, + "reward_change_max": 0.0, + "reward_change_mean": -0.4116574004292488, + "reward_change_min": -0.6464647352695465, + "reward_change_std": 0.253952544182539, + "reward_std": 0.6010984499007463, + "rewards/accuracy_reward": 0.37500000931322575, + "rewards/cosine_scaled_reward": 0.09215660532936454, + "step": 413 + }, + { + "clip_fraction": 0.0, + "completion_length": 2948.562530517578, + "epoch": 0.47314285714285714, + "grad_norm": 0.06618716567754745, + "kl": 1.0458752512931824e-05, + "lambda_div_used": 0.6192215830087662, + "learning_rate": 1.804828558898332e-07, + "loss": -0.0483, + "reward": -0.21263186633586884, + "reward_after_mean": -0.21263186633586884, + "reward_after_std": 0.5998369809240103, + "reward_before_mean": 0.04798411298543215, + "reward_before_std": 0.555210480466485, + "reward_change_max": 0.0, + "reward_change_mean": -0.2606159746646881, + "reward_change_min": -0.4564817361533642, + "reward_change_std": 0.16899613942950964, + "reward_std": 0.5998369976878166, + "rewards/accuracy_reward": 0.1458333358168602, + "rewards/cosine_scaled_reward": -0.0978492172434926, + "step": 414 + }, + { + "clip_fraction": 0.0, + "completion_length": 3234.8125, + "epoch": 0.4742857142857143, + "grad_norm": 0.06085793301463127, + "kl": 2.8954818844795227e-05, + "lambda_div_used": 0.5570018589496613, + "learning_rate": 1.7869892577476722e-07, + "loss": 0.0073, + "reward": -0.46066740388050675, + "reward_after_mean": -0.46066740388050675, + "reward_after_std": 0.3419057931751013, + "reward_before_mean": -0.2106443401426077, + "reward_before_std": 0.2546057654544711, + "reward_change_max": 0.0, + "reward_change_mean": -0.2500230632722378, + "reward_change_min": -0.36453111097216606, + "reward_change_std": 0.1324605904519558, + "reward_std": 0.3419058118015528, + "rewards/accuracy_reward": 0.02083333395421505, + "rewards/cosine_scaled_reward": -0.23147768154740334, + "step": 415 + }, + { + "clip_fraction": 0.0, + "completion_length": 1748.9583892822266, + "epoch": 0.4754285714285714, + "grad_norm": 0.09120064228773117, + "kl": 8.981674909591675e-06, + "lambda_div_used": 0.6067373231053352, + "learning_rate": 1.7693309235023127e-07, + "loss": -0.0771, + "reward": -0.13937147613614798, + "reward_after_mean": -0.13937147613614798, + "reward_after_std": 0.6240962240844965, + "reward_before_mean": 0.20647221896797419, + "reward_before_std": 0.48884215706493706, + "reward_change_max": 0.0, + "reward_change_mean": -0.34584368020296097, + "reward_change_min": -0.5141387544572353, + "reward_change_std": 0.18731264490634203, + "reward_std": 0.6240962333977222, + "rewards/accuracy_reward": 0.2500000037252903, + "rewards/cosine_scaled_reward": -0.04352780181216076, + "step": 416 + }, + { + "clip_fraction": 0.0, + "completion_length": 3307.250015258789, + "epoch": 0.4765714285714286, + "grad_norm": 0.05766940861940384, + "kl": 3.1495699658989906e-05, + "lambda_div_used": 0.5544050261378288, + "learning_rate": 1.7518544168045524e-07, + "loss": -0.0019, + "reward": -0.4980150870978832, + "reward_after_mean": -0.4980150870978832, + "reward_after_std": 0.3123048096895218, + "reward_before_mean": -0.2511206082999706, + "reward_before_std": 0.24326619878411293, + "reward_change_max": 0.0, + "reward_change_mean": -0.24689447320997715, + "reward_change_min": -0.3710709176957607, + "reward_change_std": 0.13346257898956537, + "reward_std": 0.3123048171401024, + "rewards/accuracy_reward": 0.02083333395421505, + "rewards/cosine_scaled_reward": -0.2719539441168308, + "step": 417 + }, + { + "clip_fraction": 0.0, + "completion_length": 2171.625015258789, + "epoch": 0.4777142857142857, + "grad_norm": 0.0854194164276123, + "kl": 2.8558075428009033e-05, + "lambda_div_used": 0.6033797562122345, + "learning_rate": 1.7345605894346726e-07, + "loss": 0.0073, + "reward": -0.0949536501429975, + "reward_after_mean": -0.0949536501429975, + "reward_after_std": 0.540143633261323, + "reward_before_mean": 0.25413690507411957, + "reward_before_std": 0.4725718079134822, + "reward_change_max": 0.0, + "reward_change_mean": -0.34909053333103657, + "reward_change_min": -0.5411153584718704, + "reward_change_std": 0.2063078135251999, + "reward_std": 0.5401436407119036, + "rewards/accuracy_reward": 0.29166667349636555, + "rewards/cosine_scaled_reward": -0.03752976981922984, + "step": 418 + }, + { + "clip_fraction": 0.0, + "completion_length": 2464.3750228881836, + "epoch": 0.47885714285714287, + "grad_norm": 0.08315848559141159, + "kl": 3.0182301998138428e-05, + "lambda_div_used": 0.5937136337161064, + "learning_rate": 1.7174502842694212e-07, + "loss": 0.03, + "reward": -0.27206650376319885, + "reward_after_mean": -0.27206650376319885, + "reward_after_std": 0.49108788557350636, + "reward_before_mean": 0.0038498505018651485, + "reward_before_std": 0.4298001816496253, + "reward_change_max": 0.0, + "reward_change_mean": -0.27591635659337044, + "reward_change_min": -0.4215012192726135, + "reward_change_std": 0.16396540496498346, + "reward_std": 0.49108790047466755, + "rewards/accuracy_reward": 0.1250000037252903, + "rewards/cosine_scaled_reward": -0.12115013878792524, + "step": 419 + }, + { + "clip_fraction": 0.0, + "completion_length": 1972.7083625793457, + "epoch": 0.48, + "grad_norm": 0.11145550012588501, + "kl": 3.493949770927429e-05, + "lambda_div_used": 0.5797194987535477, + "learning_rate": 1.7005243352409333e-07, + "loss": -0.0607, + "reward": -0.1652970388531685, + "reward_after_mean": -0.1652970388531685, + "reward_after_std": 0.4477591011673212, + "reward_before_mean": 0.18980162939988077, + "reward_before_std": 0.36517443507909775, + "reward_change_max": 0.0, + "reward_change_mean": -0.35509867407381535, + "reward_change_min": -0.5067496970295906, + "reward_change_std": 0.19984195847064257, + "reward_std": 0.4477591197937727, + "rewards/accuracy_reward": 0.22916666977107525, + "rewards/cosine_scaled_reward": -0.03936504106968641, + "step": 420 + }, + { + "clip_fraction": 0.0, + "completion_length": 3191.937515258789, + "epoch": 0.48114285714285715, + "grad_norm": 0.07170204073190689, + "kl": 3.11434268951416e-05, + "lambda_div_used": 0.5753046199679375, + "learning_rate": 1.6837835672960831e-07, + "loss": 0.0088, + "reward": -0.38796948781237006, + "reward_after_mean": -0.38796948781237006, + "reward_after_std": 0.4225973077118397, + "reward_before_mean": -0.12759371474385262, + "reward_before_std": 0.3405588921159506, + "reward_change_max": 0.0, + "reward_change_mean": -0.2603757604956627, + "reward_change_min": -0.37436668202281, + "reward_change_std": 0.13996374886482954, + "reward_std": 0.4225973132997751, + "rewards/accuracy_reward": 0.0416666679084301, + "rewards/cosine_scaled_reward": -0.16926039289683104, + "step": 421 + }, + { + "clip_fraction": 0.0, + "completion_length": 2776.437545776367, + "epoch": 0.48228571428571426, + "grad_norm": 0.06681375205516815, + "kl": 1.9788742065429688e-05, + "lambda_div_used": 0.5904401987791061, + "learning_rate": 1.6672287963562852e-07, + "loss": 0.0293, + "reward": -0.25396816432476044, + "reward_after_mean": -0.25396816432476044, + "reward_after_std": 0.4787884410470724, + "reward_before_mean": 0.03486193250864744, + "reward_before_std": 0.4093271289020777, + "reward_change_max": 0.0, + "reward_change_mean": -0.2888300847262144, + "reward_change_min": -0.43693453073501587, + "reward_change_std": 0.16317449882626534, + "reward_std": 0.4787884559482336, + "rewards/accuracy_reward": 0.1458333395421505, + "rewards/cosine_scaled_reward": -0.11097141215577722, + "step": 422 + }, + { + "clip_fraction": 0.0, + "completion_length": 2945.9583740234375, + "epoch": 0.48342857142857143, + "grad_norm": 0.06767871230840683, + "kl": 2.205371856689453e-05, + "lambda_div_used": 0.611551009118557, + "learning_rate": 1.6508608292777203e-07, + "loss": 0.0153, + "reward": -0.16482537053525448, + "reward_after_mean": -0.16482537053525448, + "reward_after_std": 0.5363675616681576, + "reward_before_mean": 0.12604539189487696, + "reward_before_std": 0.518328445032239, + "reward_change_max": 0.0, + "reward_change_mean": -0.2908707782626152, + "reward_change_min": -0.4775175042450428, + "reward_change_std": 0.18889948446303606, + "reward_std": 0.5363675802946091, + "rewards/accuracy_reward": 0.16666666977107525, + "rewards/cosine_scaled_reward": -0.040621266700327396, + "step": 423 + }, + { + "clip_fraction": 0.0, + "completion_length": 3215.0208740234375, + "epoch": 0.4845714285714286, + "grad_norm": 0.061716482043266296, + "kl": 1.1460855603218079e-05, + "lambda_div_used": 0.5937408357858658, + "learning_rate": 1.6346804638120098e-07, + "loss": 0.0327, + "reward": -0.3415182586759329, + "reward_after_mean": -0.3415182586759329, + "reward_after_std": 0.48476750776171684, + "reward_before_mean": -0.08947536488994956, + "reward_before_std": 0.4272408355027437, + "reward_change_max": 0.0, + "reward_change_mean": -0.2520429063588381, + "reward_change_min": -0.42084335163235664, + "reward_change_std": 0.15120856929570436, + "reward_std": 0.4847675133496523, + "rewards/accuracy_reward": 0.0833333358168602, + "rewards/cosine_scaled_reward": -0.17280870315153152, + "step": 424 + }, + { + "clip_fraction": 0.0, + "completion_length": 2154.9375610351562, + "epoch": 0.4857142857142857, + "grad_norm": 0.09371069073677063, + "kl": 3.830809146165848e-05, + "lambda_div_used": 0.6315138638019562, + "learning_rate": 1.6186884885673413e-07, + "loss": 0.0058, + "reward": 0.34828917868435383, + "reward_after_mean": 0.34828917868435383, + "reward_after_std": 0.7491090279072523, + "reward_before_mean": 0.91605463065207, + "reward_before_std": 0.6105400957167149, + "reward_change_max": 0.0, + "reward_change_mean": -0.5677654705941677, + "reward_change_min": -0.9185826852917671, + "reward_change_std": 0.34253316558897495, + "reward_std": 0.7491090428084135, + "rewards/accuracy_reward": 0.5833333358168602, + "rewards/cosine_scaled_reward": 0.3327212668955326, + "step": 425 + }, + { + "clip_fraction": 0.0, + "completion_length": 2205.3125190734863, + "epoch": 0.4868571428571429, + "grad_norm": 0.09298980236053467, + "kl": 2.3175030946731567e-05, + "lambda_div_used": 0.6215192526578903, + "learning_rate": 1.6028856829700258e-07, + "loss": -0.002, + "reward": -0.029936233535408974, + "reward_after_mean": -0.029936233535408974, + "reward_after_std": 0.5604026541113853, + "reward_before_mean": 0.3139693345874548, + "reward_before_std": 0.5635973755270243, + "reward_change_max": 0.0, + "reward_change_mean": -0.34390556812286377, + "reward_change_min": -0.5907084755599499, + "reward_change_std": 0.23498705588281155, + "reward_std": 0.5604026671499014, + "rewards/accuracy_reward": 0.3125000074505806, + "rewards/cosine_scaled_reward": 0.0014693308621644974, + "step": 426 + }, + { + "clip_fraction": 0.0, + "completion_length": 3295.7708740234375, + "epoch": 0.488, + "grad_norm": 0.0548052154481411, + "kl": 2.4527311325073242e-05, + "lambda_div_used": 0.5990054532885551, + "learning_rate": 1.5872728172265146e-07, + "loss": 0.0088, + "reward": -0.2071160115301609, + "reward_after_mean": -0.2071160115301609, + "reward_after_std": 0.49672279693186283, + "reward_before_mean": 0.08893106225878, + "reward_before_std": 0.454012256115675, + "reward_change_max": 0.0, + "reward_change_mean": -0.2960470784455538, + "reward_change_min": -0.4723533205688, + "reward_change_std": 0.18442231137305498, + "reward_std": 0.4967228155583143, + "rewards/accuracy_reward": 0.18750000558793545, + "rewards/cosine_scaled_reward": -0.09856893587857485, + "step": 427 + }, + { + "clip_fraction": 0.0, + "completion_length": 2302.520881652832, + "epoch": 0.48914285714285716, + "grad_norm": 0.11483462899923325, + "kl": 0.000337366946041584, + "lambda_div_used": 0.5547136962413788, + "learning_rate": 1.5718506522858572e-07, + "loss": 0.0311, + "reward": -0.42663199454545975, + "reward_after_mean": -0.42663199454545975, + "reward_after_std": 0.29241302236914635, + "reward_before_mean": -0.1547260768711567, + "reward_before_std": 0.24498768709599972, + "reward_change_max": 0.0, + "reward_change_mean": -0.27190591767430305, + "reward_change_min": -0.42438486963510513, + "reward_change_std": 0.1579811777919531, + "reward_std": 0.2924130354076624, + "rewards/accuracy_reward": 0.0833333358168602, + "rewards/cosine_scaled_reward": -0.2380594089627266, + "step": 428 + }, + { + "clip_fraction": 0.0, + "completion_length": 2431.520881652832, + "epoch": 0.49028571428571427, + "grad_norm": 0.0878632664680481, + "kl": 2.4685636162757874e-05, + "lambda_div_used": 0.6025609895586967, + "learning_rate": 1.5566199398026147e-07, + "loss": 0.0378, + "reward": -0.057155030546709895, + "reward_after_mean": -0.057155030546709895, + "reward_after_std": 0.5567535478621721, + "reward_before_mean": 0.3208682704716921, + "reward_before_std": 0.47341752983629704, + "reward_change_max": 0.0, + "reward_change_mean": -0.3780232574790716, + "reward_change_min": -0.5763550102710724, + "reward_change_std": 0.22364642471075058, + "reward_std": 0.5567535553127527, + "rewards/accuracy_reward": 0.29166666977107525, + "rewards/cosine_scaled_reward": 0.029201554832980037, + "step": 429 + }, + { + "clip_fraction": 0.0, + "completion_length": 2577.3125610351562, + "epoch": 0.49142857142857144, + "grad_norm": 0.08103640377521515, + "kl": 1.4763325452804565e-05, + "lambda_div_used": 0.6012379750609398, + "learning_rate": 1.5415814221002265e-07, + "loss": -0.0049, + "reward": 0.031602535396814346, + "reward_after_mean": 0.031602535396814346, + "reward_after_std": 0.5440461356192827, + "reward_before_mean": 0.4564003311097622, + "reward_before_std": 0.46839726250618696, + "reward_change_max": 0.0, + "reward_change_mean": -0.42479780688881874, + "reward_change_min": -0.6648109555244446, + "reward_change_std": 0.2579617351293564, + "reward_std": 0.5440461542457342, + "rewards/accuracy_reward": 0.3541666753590107, + "rewards/cosine_scaled_reward": 0.1022336557507515, + "step": 430 + }, + { + "clip_fraction": 0.0, + "completion_length": 2690.6458740234375, + "epoch": 0.49257142857142855, + "grad_norm": 0.06538030505180359, + "kl": 2.8170645236968994e-05, + "lambda_div_used": 0.5918548330664635, + "learning_rate": 1.5267358321348285e-07, + "loss": -0.0328, + "reward": -0.24143310775980353, + "reward_after_mean": -0.24143310775980353, + "reward_after_std": 0.481786971911788, + "reward_before_mean": 0.05766227189451456, + "reward_before_std": 0.41715206764638424, + "reward_change_max": 0.0, + "reward_change_mean": -0.29909538850188255, + "reward_change_min": -0.4450513869524002, + "reward_change_std": 0.1692206682637334, + "reward_std": 0.4817869979888201, + "rewards/accuracy_reward": 0.1458333395421505, + "rewards/cosine_scaled_reward": -0.08817105047637597, + "step": 431 + }, + { + "clip_fraction": 0.0, + "completion_length": 2843.750045776367, + "epoch": 0.4937142857142857, + "grad_norm": 0.05780716612935066, + "kl": 3.104284405708313e-05, + "lambda_div_used": 0.576793298125267, + "learning_rate": 1.5120838934595337e-07, + "loss": 0.0053, + "reward": -0.42155745439231396, + "reward_after_mean": -0.42155745439231396, + "reward_after_std": 0.4230448566377163, + "reward_before_mean": -0.19034346495755017, + "reward_before_std": 0.35024640895426273, + "reward_change_max": 0.0, + "reward_change_mean": -0.23121398873627186, + "reward_change_min": -0.3715337961912155, + "reward_change_std": 0.1338915005326271, + "reward_std": 0.4230448678135872, + "rewards/accuracy_reward": 0.06250000186264515, + "rewards/cosine_scaled_reward": -0.2528434665873647, + "step": 432 + }, + { + "clip_fraction": 0.0, + "completion_length": 2992.062530517578, + "epoch": 0.4948571428571429, + "grad_norm": 0.06562553346157074, + "kl": 3.0972063541412354e-05, + "lambda_div_used": 0.6079665347933769, + "learning_rate": 1.4976263201891613e-07, + "loss": 0.1201, + "reward": -0.26266857516020536, + "reward_after_mean": -0.26266857516020536, + "reward_after_std": 0.5306479204446077, + "reward_before_mean": -0.0102414321154356, + "reward_before_std": 0.5030816271901131, + "reward_change_max": 0.0, + "reward_change_mean": -0.25242713652551174, + "reward_change_min": -0.4955142140388489, + "reward_change_std": 0.1781205264851451, + "reward_std": 0.5306479260325432, + "rewards/accuracy_reward": 0.1458333358168602, + "rewards/cosine_scaled_reward": -0.15607476886361837, + "step": 433 + }, + { + "clip_fraction": 0.0, + "completion_length": 2915.625030517578, + "epoch": 0.496, + "grad_norm": 0.06934478878974915, + "kl": 3.8422178477048874e-05, + "lambda_div_used": 0.5604980885982513, + "learning_rate": 1.483363816965435e-07, + "loss": -0.004, + "reward": -0.4280230412259698, + "reward_after_mean": -0.4280230412259698, + "reward_after_std": 0.32627478428184986, + "reward_before_mean": -0.15262745507061481, + "reward_before_std": 0.2731653768569231, + "reward_change_max": 0.0, + "reward_change_mean": -0.27539557591080666, + "reward_change_min": -0.44822341948747635, + "reward_change_std": 0.16339552495628595, + "reward_std": 0.32627478800714016, + "rewards/accuracy_reward": 0.0416666679084301, + "rewards/cosine_scaled_reward": -0.1942941201850772, + "step": 434 + }, + { + "clip_fraction": 0.0, + "completion_length": 2320.750026702881, + "epoch": 0.49714285714285716, + "grad_norm": 0.13675029575824738, + "kl": 4.8140063881874084e-05, + "lambda_div_used": 0.5811168253421783, + "learning_rate": 1.469297078922642e-07, + "loss": -0.0374, + "reward": -0.17623315937817097, + "reward_after_mean": -0.17623315937817097, + "reward_after_std": 0.483237961307168, + "reward_before_mean": 0.19585114251822233, + "reward_before_std": 0.3759047882631421, + "reward_change_max": 0.0, + "reward_change_mean": -0.37208431400358677, + "reward_change_min": -0.5468379594385624, + "reward_change_std": 0.20962903555482626, + "reward_std": 0.4832379762083292, + "rewards/accuracy_reward": 0.20833333395421505, + "rewards/cosine_scaled_reward": -0.01248218398541212, + "step": 435 + }, + { + "clip_fraction": 0.0, + "completion_length": 2316.3125381469727, + "epoch": 0.4982857142857143, + "grad_norm": 0.09783437103033066, + "kl": 2.839416265487671e-05, + "lambda_div_used": 0.6289880573749542, + "learning_rate": 1.4554267916537495e-07, + "loss": 0.0826, + "reward": 0.12041758373379707, + "reward_after_mean": 0.12041758373379707, + "reward_after_std": 0.6426001656800508, + "reward_before_mean": 0.5348608233034611, + "reward_before_std": 0.6021685730665922, + "reward_change_max": 0.0, + "reward_change_mean": -0.41444322280585766, + "reward_change_min": -0.6993351243436337, + "reward_change_std": 0.271469890139997, + "reward_std": 0.6426001694053411, + "rewards/accuracy_reward": 0.3958333395421505, + "rewards/cosine_scaled_reward": 0.13902746886014938, + "step": 436 + }, + { + "clip_fraction": 0.0, + "completion_length": 2954.0416717529297, + "epoch": 0.49942857142857144, + "grad_norm": 0.05496148392558098, + "kl": 0.00023943744599819183, + "lambda_div_used": 0.5626013651490211, + "learning_rate": 1.4417536311769885e-07, + "loss": 0.05, + "reward": -0.42826351523399353, + "reward_after_mean": -0.42826351523399353, + "reward_after_std": 0.33650987036526203, + "reward_before_mean": -0.16868688352406025, + "reward_before_std": 0.2832178361713886, + "reward_change_max": 0.0, + "reward_change_mean": -0.25957662612199783, + "reward_change_min": -0.42033713683485985, + "reward_change_std": 0.1560918828472495, + "reward_std": 0.3365098759531975, + "rewards/accuracy_reward": 0.0625, + "rewards/cosine_scaled_reward": -0.2311868779361248, + "step": 437 + }, + { + "clip_fraction": 0.0, + "completion_length": 3007.500015258789, + "epoch": 0.5005714285714286, + "grad_norm": 0.05682806298136711, + "kl": 4.358415026217699e-06, + "lambda_div_used": 0.6039799600839615, + "learning_rate": 1.4282782639029128e-07, + "loss": 0.0289, + "reward": -0.29300207551568747, + "reward_after_mean": -0.29300207551568747, + "reward_after_std": 0.5204536523669958, + "reward_before_mean": -0.03094739466905594, + "reward_before_std": 0.4829921592026949, + "reward_change_max": 0.0, + "reward_change_mean": -0.2620546855032444, + "reward_change_min": -0.5145607963204384, + "reward_change_std": 0.18371508549898863, + "reward_std": 0.5204536579549313, + "rewards/accuracy_reward": 0.1250000037252903, + "rewards/cosine_scaled_reward": -0.15594739385414869, + "step": 438 + }, + { + "clip_fraction": 0.0, + "completion_length": 2249.5000076293945, + "epoch": 0.5017142857142857, + "grad_norm": 0.0828532725572586, + "kl": 3.217160701751709e-05, + "lambda_div_used": 0.5708463862538338, + "learning_rate": 1.4150013466019114e-07, + "loss": 0.021, + "reward": -0.40206424333155155, + "reward_after_mean": -0.40206424333155155, + "reward_after_std": 0.40689974650740623, + "reward_before_mean": -0.14063972979784012, + "reward_before_std": 0.32079045102000237, + "reward_change_max": 0.0, + "reward_change_mean": -0.261424507945776, + "reward_change_min": -0.38026969507336617, + "reward_change_std": 0.14067976083606482, + "reward_std": 0.406899768859148, + "rewards/accuracy_reward": 0.0416666679084301, + "rewards/cosine_scaled_reward": -0.1823064020718448, + "step": 439 + }, + { + "clip_fraction": 0.0, + "completion_length": 2964.000011444092, + "epoch": 0.5028571428571429, + "grad_norm": 0.07350306212902069, + "kl": 2.7701258659362793e-05, + "lambda_div_used": 0.5365751013159752, + "learning_rate": 1.4019235263722034e-07, + "loss": 0.0201, + "reward": -0.5367953963577747, + "reward_after_mean": -0.5367953963577747, + "reward_after_std": 0.231884878128767, + "reward_before_mean": -0.28081973269581795, + "reward_before_std": 0.16059848852455616, + "reward_change_max": 0.0, + "reward_change_mean": -0.25597566179931164, + "reward_change_min": -0.38940757140517235, + "reward_change_std": 0.13862022012472153, + "reward_std": 0.2318848893046379, + "rewards/accuracy_reward": 0.0, + "rewards/cosine_scaled_reward": -0.28081973642110825, + "step": 440 + }, + { + "clip_fraction": 0.0, + "completion_length": 3007.770896911621, + "epoch": 0.504, + "grad_norm": 0.07498018443584442, + "kl": 2.944841980934143e-05, + "lambda_div_used": 0.6011143997311592, + "learning_rate": 1.3890454406082956e-07, + "loss": -0.0116, + "reward": -0.13889071717858315, + "reward_after_mean": -0.13889071717858315, + "reward_after_std": 0.4864419028162956, + "reward_before_mean": 0.17084867507219315, + "reward_before_std": 0.4689741334877908, + "reward_change_max": 0.0, + "reward_change_mean": -0.3097394183278084, + "reward_change_min": -0.514279019087553, + "reward_change_std": 0.20097953081130981, + "reward_std": 0.48644191212952137, + "rewards/accuracy_reward": 0.2291666753590107, + "rewards/cosine_scaled_reward": -0.05831799004226923, + "step": 441 + }, + { + "clip_fraction": 0.0, + "completion_length": 2837.4583435058594, + "epoch": 0.5051428571428571, + "grad_norm": 0.0673719272017479, + "kl": 3.160163760185242e-05, + "lambda_div_used": 0.5689455196261406, + "learning_rate": 1.3763677169699217e-07, + "loss": 0.0084, + "reward": -0.31079866108484566, + "reward_after_mean": -0.31079866108484566, + "reward_after_std": 0.3982803635299206, + "reward_before_mean": -0.010625829687342048, + "reward_before_std": 0.31661996035836637, + "reward_change_max": 0.0, + "reward_change_mean": -0.30017283000051975, + "reward_change_min": -0.4261031448841095, + "reward_change_std": 0.16800945159047842, + "reward_std": 0.3982803765684366, + "rewards/accuracy_reward": 0.10416666977107525, + "rewards/cosine_scaled_reward": -0.1147925069089979, + "step": 442 + }, + { + "clip_fraction": 0.0, + "completion_length": 3194.270854949951, + "epoch": 0.5062857142857143, + "grad_norm": 0.05984271690249443, + "kl": 1.745857298374176e-05, + "lambda_div_used": 0.5607094466686249, + "learning_rate": 1.3638909733514452e-07, + "loss": 0.0053, + "reward": -0.38279488682746887, + "reward_after_mean": -0.38279488682746887, + "reward_after_std": 0.33906700275838375, + "reward_before_mean": -0.09817820321768522, + "reward_before_std": 0.2764374865218997, + "reward_change_max": 0.0, + "reward_change_mean": -0.2846166864037514, + "reward_change_min": -0.4438001811504364, + "reward_change_std": 0.16594033408910036, + "reward_std": 0.3390670083463192, + "rewards/accuracy_reward": 0.0625, + "rewards/cosine_scaled_reward": -0.16067820694297552, + "step": 443 + }, + { + "clip_fraction": 0.0, + "completion_length": 2902.2291717529297, + "epoch": 0.5074285714285715, + "grad_norm": 0.0638304203748703, + "kl": 1.5033408999443054e-05, + "lambda_div_used": 0.5774018093943596, + "learning_rate": 1.351615817851748e-07, + "loss": -0.0355, + "reward": -0.3352237604558468, + "reward_after_mean": -0.3352237604558468, + "reward_after_std": 0.4039857666939497, + "reward_before_mean": -0.05285666696727276, + "reward_before_std": 0.35096561443060637, + "reward_change_max": 0.0, + "reward_change_mean": -0.2823671065270901, + "reward_change_min": -0.42594215646386147, + "reward_change_std": 0.16656992863863707, + "reward_std": 0.40398577041924, + "rewards/accuracy_reward": 0.10416666977107525, + "rewards/cosine_scaled_reward": -0.15702332742512226, + "step": 444 + }, + { + "clip_fraction": 0.0, + "completion_length": 3006.916717529297, + "epoch": 0.5085714285714286, + "grad_norm": 0.06397882103919983, + "kl": 2.842303365468979e-05, + "lambda_div_used": 0.6194909662008286, + "learning_rate": 1.3395428487445914e-07, + "loss": 0.0715, + "reward": -0.05263347551226616, + "reward_after_mean": -0.05263347551226616, + "reward_after_std": 0.5679135732352734, + "reward_before_mean": 0.2764017879962921, + "reward_before_std": 0.5510673765093088, + "reward_change_max": 0.0, + "reward_change_mean": -0.3290352523326874, + "reward_change_min": -0.5197134874761105, + "reward_change_std": 0.21186745446175337, + "reward_std": 0.5679135844111443, + "rewards/accuracy_reward": 0.27083334140479565, + "rewards/cosine_scaled_reward": 0.0055684298276901245, + "step": 445 + }, + { + "clip_fraction": 0.0, + "completion_length": 3028.8333892822266, + "epoch": 0.5097142857142857, + "grad_norm": 0.06140498071908951, + "kl": 3.2648444175720215e-05, + "lambda_div_used": 0.606703408062458, + "learning_rate": 1.3276726544494571e-07, + "loss": -0.0241, + "reward": -0.2073356769979, + "reward_after_mean": -0.2073356769979, + "reward_after_std": 0.5240648984909058, + "reward_before_mean": 0.0723939798772335, + "reward_before_std": 0.4952498711645603, + "reward_change_max": 0.0, + "reward_change_mean": -0.27972964756190777, + "reward_change_min": -0.5379137881100178, + "reward_change_std": 0.1909602265805006, + "reward_std": 0.5240649078041315, + "rewards/accuracy_reward": 0.1458333358168602, + "rewards/cosine_scaled_reward": -0.07343936339020729, + "step": 446 + }, + { + "clip_fraction": 0.0, + "completion_length": 2571.166721343994, + "epoch": 0.5108571428571429, + "grad_norm": 0.08778514713048935, + "kl": 9.072478860616684e-06, + "lambda_div_used": 0.5970501974225044, + "learning_rate": 1.316005813502869e-07, + "loss": 0.0018, + "reward": -0.19514422863721848, + "reward_after_mean": -0.19514422863721848, + "reward_after_std": 0.4898754768073559, + "reward_before_mean": 0.10967571474611759, + "reward_before_std": 0.4461768325418234, + "reward_change_max": 0.0, + "reward_change_mean": -0.3048199340701103, + "reward_change_min": -0.48957687243819237, + "reward_change_std": 0.18721044715493917, + "reward_std": 0.4898754861205816, + "rewards/accuracy_reward": 0.18750000558793545, + "rewards/cosine_scaled_reward": -0.07782429456710815, + "step": 447 + }, + { + "clip_fraction": 0.0, + "completion_length": 2039.6042022705078, + "epoch": 0.512, + "grad_norm": 0.12122303992509842, + "kl": 3.530830144882202e-05, + "lambda_div_used": 0.5591797679662704, + "learning_rate": 1.3045428945301953e-07, + "loss": 0.0057, + "reward": -0.3564775697886944, + "reward_after_mean": -0.3564775697886944, + "reward_after_std": 0.33722602762281895, + "reward_before_mean": -0.06660962291061878, + "reward_before_std": 0.27086107339709997, + "reward_change_max": 0.0, + "reward_change_mean": -0.28986795619130135, + "reward_change_min": -0.4345347508788109, + "reward_change_std": 0.1656003799289465, + "reward_std": 0.33722603134810925, + "rewards/accuracy_reward": 0.0833333358168602, + "rewards/cosine_scaled_reward": -0.14994295500218868, + "step": 448 + }, + { + "clip_fraction": 0.0, + "completion_length": 2680.0208435058594, + "epoch": 0.5131428571428571, + "grad_norm": 0.07745695859193802, + "kl": 3.194063901901245e-05, + "lambda_div_used": 0.5827131420373917, + "learning_rate": 1.2932844562179352e-07, + "loss": -0.0247, + "reward": -0.278532937169075, + "reward_after_mean": -0.278532937169075, + "reward_after_std": 0.44566163793206215, + "reward_before_mean": 0.01611769199371338, + "reward_before_std": 0.3775772461667657, + "reward_change_max": 0.0, + "reward_change_mean": -0.29465061984956264, + "reward_change_min": -0.4545559212565422, + "reward_change_std": 0.17489958554506302, + "reward_std": 0.44566163793206215, + "rewards/accuracy_reward": 0.18750000558793545, + "rewards/cosine_scaled_reward": -0.17138232477009296, + "step": 449 + }, + { + "clip_fraction": 0.0, + "completion_length": 2405.0833702087402, + "epoch": 0.5142857142857142, + "grad_norm": 0.07359456270933151, + "kl": 4.271417856216431e-05, + "lambda_div_used": 0.5313882529735565, + "learning_rate": 1.2822310472864885e-07, + "loss": 0.0047, + "reward": -0.40455422177910805, + "reward_after_mean": -0.40455422177910805, + "reward_after_std": 0.27055008336901665, + "reward_before_mean": -0.05470774322748184, + "reward_before_std": 0.13779542688280344, + "reward_change_max": 0.0, + "reward_change_mean": -0.34984651021659374, + "reward_change_min": -0.482446551322937, + "reward_change_std": 0.18013106007128954, + "reward_std": 0.2705500964075327, + "rewards/accuracy_reward": 0.125, + "rewards/cosine_scaled_reward": -0.17970772832632065, + "step": 450 + }, + { + "clip_fraction": 0.0, + "completion_length": 2726.7916870117188, + "epoch": 0.5154285714285715, + "grad_norm": 0.10456772893667221, + "kl": 5.123019218444824e-05, + "lambda_div_used": 0.5659012496471405, + "learning_rate": 1.2713832064634125e-07, + "loss": 0.0437, + "reward": -0.233762688934803, + "reward_after_mean": -0.233762688934803, + "reward_after_std": 0.3937326893210411, + "reward_before_mean": 0.14449233608320355, + "reward_before_std": 0.30359079129993916, + "reward_change_max": 0.0, + "reward_change_mean": -0.3782550562173128, + "reward_change_min": -0.5748258531093597, + "reward_change_std": 0.21745321340858936, + "reward_std": 0.3937326930463314, + "rewards/accuracy_reward": 0.1875, + "rewards/cosine_scaled_reward": -0.0430076620541513, + "step": 451 + }, + { + "clip_fraction": 0.0, + "completion_length": 3173.812530517578, + "epoch": 0.5165714285714286, + "grad_norm": 0.05382152646780014, + "kl": 1.621060073375702e-05, + "lambda_div_used": 0.628274716436863, + "learning_rate": 1.260741462457165e-07, + "loss": 0.0152, + "reward": 0.08829959481954575, + "reward_after_mean": 0.08829959481954575, + "reward_after_std": 0.6491915434598923, + "reward_before_mean": 0.49067158019170165, + "reward_before_std": 0.5948440972715616, + "reward_change_max": 0.0, + "reward_change_mean": -0.4023719858378172, + "reward_change_min": -0.6429549157619476, + "reward_change_std": 0.2526876190677285, + "reward_std": 0.6491915658116341, + "rewards/accuracy_reward": 0.37500000558793545, + "rewards/cosine_scaled_reward": 0.11567158252000809, + "step": 452 + }, + { + "clip_fraction": 0.0, + "completion_length": 2601.1666870117188, + "epoch": 0.5177142857142857, + "grad_norm": 0.10710335522890091, + "kl": 4.57763671875e-05, + "lambda_div_used": 0.5769856572151184, + "learning_rate": 1.2503063339313356e-07, + "loss": -0.0233, + "reward": -0.4120060931891203, + "reward_after_mean": -0.4120060931891203, + "reward_after_std": 0.4247955661267042, + "reward_before_mean": -0.17395742796361446, + "reward_before_std": 0.3489131908863783, + "reward_change_max": 0.0, + "reward_change_mean": -0.23804865032434464, + "reward_change_min": -0.3744778670370579, + "reward_change_std": 0.12989939749240875, + "reward_std": 0.4247955847531557, + "rewards/accuracy_reward": 0.0416666679084301, + "rewards/cosine_scaled_reward": -0.21562409959733486, + "step": 453 + }, + { + "clip_fraction": 0.0, + "completion_length": 2794.8958740234375, + "epoch": 0.5188571428571429, + "grad_norm": 0.059399981051683426, + "kl": 1.829490065574646e-05, + "lambda_div_used": 0.604412779211998, + "learning_rate": 1.2400783294793668e-07, + "loss": 0.0084, + "reward": -0.23108407109975815, + "reward_after_mean": -0.23108407109975815, + "reward_after_std": 0.5207228269428015, + "reward_before_mean": 0.05918463226407766, + "reward_before_std": 0.4853199487552047, + "reward_change_max": 0.0, + "reward_change_mean": -0.29026869870722294, + "reward_change_min": -0.5276229903101921, + "reward_change_std": 0.1960765514522791, + "reward_std": 0.5207228306680918, + "rewards/accuracy_reward": 0.1458333358168602, + "rewards/cosine_scaled_reward": -0.08664871653309092, + "step": 454 + }, + { + "clip_fraction": 0.0, + "completion_length": 3120.0416717529297, + "epoch": 0.52, + "grad_norm": 0.07823975384235382, + "kl": 1.1070631444454193e-05, + "lambda_div_used": 0.5381916239857674, + "learning_rate": 1.2300579475997657e-07, + "loss": 0.0035, + "reward": -0.5376988351345062, + "reward_after_mean": -0.5376988351345062, + "reward_after_std": 0.2399882897734642, + "reward_before_mean": -0.28268107399344444, + "reward_before_std": 0.16770172398537397, + "reward_change_max": 0.0, + "reward_change_mean": -0.25501775927841663, + "reward_change_min": -0.38403724879026413, + "reward_change_std": 0.1382020702585578, + "reward_std": 0.23998829536139965, + "rewards/accuracy_reward": 0.0, + "rewards/cosine_scaled_reward": -0.28268107026815414, + "step": 455 + }, + { + "clip_fraction": 0.0, + "completion_length": 3183.125, + "epoch": 0.5211428571428571, + "grad_norm": 0.07441714406013489, + "kl": 1.7248094081878662e-05, + "lambda_div_used": 0.535825714468956, + "learning_rate": 1.220245676671809e-07, + "loss": -0.0061, + "reward": -0.5242688432335854, + "reward_after_mean": -0.5242688432335854, + "reward_after_std": 0.2355129700154066, + "reward_before_mean": -0.2603513076901436, + "reward_before_std": 0.15733365854248405, + "reward_change_max": 0.0, + "reward_change_mean": -0.2639175299555063, + "reward_change_min": -0.3936190530657768, + "reward_change_std": 0.14126356784254313, + "reward_std": 0.23551297932863235, + "rewards/accuracy_reward": 0.0, + "rewards/cosine_scaled_reward": -0.2603513114154339, + "step": 456 + }, + { + "clip_fraction": 0.0, + "completion_length": 3090.9166717529297, + "epoch": 0.5222857142857142, + "grad_norm": 0.06740739941596985, + "kl": 1.6771256923675537e-05, + "lambda_div_used": 0.5661213397979736, + "learning_rate": 1.2106419949317388e-07, + "loss": 0.0183, + "reward": -0.2666686773300171, + "reward_after_mean": -0.2666686773300171, + "reward_after_std": 0.3630063198506832, + "reward_before_mean": 0.06939902156591415, + "reward_before_std": 0.3003841144964099, + "reward_change_max": 0.0, + "reward_change_mean": -0.3360677044838667, + "reward_change_min": -0.5109980814158916, + "reward_change_std": 0.19570716377347708, + "reward_std": 0.36300632916390896, + "rewards/accuracy_reward": 0.1458333395421505, + "rewards/cosine_scaled_reward": -0.0764343123883009, + "step": 457 + }, + { + "clip_fraction": 0.0, + "completion_length": 2337.0416717529297, + "epoch": 0.5234285714285715, + "grad_norm": 0.10038017481565475, + "kl": 1.5580561012029648e-05, + "lambda_div_used": 0.587464913725853, + "learning_rate": 1.2012473704494537e-07, + "loss": 0.0337, + "reward": -0.2921891398727894, + "reward_after_mean": -0.2921891398727894, + "reward_after_std": 0.46863692067563534, + "reward_before_mean": -0.017078701872378588, + "reward_before_std": 0.3978575337678194, + "reward_change_max": 0.0, + "reward_change_mean": -0.27511043660342693, + "reward_change_min": -0.4165792725980282, + "reward_change_std": 0.15801922790706158, + "reward_std": 0.4686369299888611, + "rewards/accuracy_reward": 0.1250000037252903, + "rewards/cosine_scaled_reward": -0.14207870024256408, + "step": 458 + }, + { + "clip_fraction": 0.0, + "completion_length": 1897.9791870117188, + "epoch": 0.5245714285714286, + "grad_norm": 0.10566994547843933, + "kl": 3.1791627407073975e-05, + "lambda_div_used": 0.6483638733625412, + "learning_rate": 1.1920622611056974e-07, + "loss": -0.0327, + "reward": 0.04763873480260372, + "reward_after_mean": 0.04763873480260372, + "reward_after_std": 0.6893943976610899, + "reward_before_mean": 0.37104589492082596, + "reward_before_std": 0.6905025038868189, + "reward_change_max": 0.0, + "reward_change_mean": -0.3234071359038353, + "reward_change_min": -0.573150984942913, + "reward_change_std": 0.22597133833914995, + "reward_std": 0.6893944274634123, + "rewards/accuracy_reward": 0.29166667349636555, + "rewards/cosine_scaled_reward": 0.07937921397387981, + "step": 459 + }, + { + "clip_fraction": 0.0, + "completion_length": 3152.9583587646484, + "epoch": 0.5257142857142857, + "grad_norm": 0.0553775392472744, + "kl": 1.6976147890090942e-05, + "lambda_div_used": 0.6212490126490593, + "learning_rate": 1.1830871145697412e-07, + "loss": -0.0403, + "reward": -0.2310712798498571, + "reward_after_mean": -0.2310712798498571, + "reward_after_std": 0.5967046339064837, + "reward_before_mean": 0.018506756518036127, + "reward_before_std": 0.5584379080682993, + "reward_change_max": 0.0, + "reward_change_mean": -0.24957803264260292, + "reward_change_min": -0.45073715783655643, + "reward_change_std": 0.16405375488102436, + "reward_std": 0.5967046469449997, + "rewards/accuracy_reward": 0.1458333358168602, + "rewards/cosine_scaled_reward": -0.1273265864001587, + "step": 460 + }, + { + "clip_fraction": 0.0, + "completion_length": 2965.625015258789, + "epoch": 0.5268571428571428, + "grad_norm": 0.0683179497718811, + "kl": 3.898143768310547e-05, + "lambda_div_used": 0.5838751494884491, + "learning_rate": 1.1743223682775649e-07, + "loss": 0.0201, + "reward": -0.36440329253673553, + "reward_after_mean": -0.36440329253673553, + "reward_after_std": 0.44049660488963127, + "reward_before_mean": -0.10854836623184383, + "reward_before_std": 0.3863721750676632, + "reward_change_max": 0.0, + "reward_change_mean": -0.25585492700338364, + "reward_change_min": -0.41783962957561016, + "reward_change_std": 0.15460436698049307, + "reward_std": 0.44049660861492157, + "rewards/accuracy_reward": 0.08333333395421505, + "rewards/cosine_scaled_reward": -0.19188169576227665, + "step": 461 + }, + { + "clip_fraction": 0.0, + "completion_length": 2874.3125076293945, + "epoch": 0.528, + "grad_norm": 0.07832667976617813, + "kl": 2.7470290660858154e-05, + "lambda_div_used": 0.5419389680027962, + "learning_rate": 1.1657684494105386e-07, + "loss": -0.0381, + "reward": -0.5123195722699165, + "reward_after_mean": -0.5123195722699165, + "reward_after_std": 0.2568210382014513, + "reward_before_mean": -0.2502432279288769, + "reward_before_std": 0.1842196974903345, + "reward_change_max": 0.0, + "reward_change_mean": -0.2620763499289751, + "reward_change_min": -0.4005816727876663, + "reward_change_std": 0.14368234388530254, + "reward_std": 0.2568210456520319, + "rewards/accuracy_reward": 0.0, + "rewards/cosine_scaled_reward": -0.2502432242035866, + "step": 462 + }, + { + "clip_fraction": 0.0, + "completion_length": 2986.8333740234375, + "epoch": 0.5291428571428571, + "grad_norm": 0.053950026631355286, + "kl": 8.771196007728577e-06, + "lambda_div_used": 0.622698076069355, + "learning_rate": 1.1574257748745986e-07, + "loss": 0.0106, + "reward": -0.2041953857988119, + "reward_after_mean": -0.2041953857988119, + "reward_after_std": 0.6069091446697712, + "reward_before_mean": 0.05050618201494217, + "reward_before_std": 0.5684067364782095, + "reward_change_max": 0.0, + "reward_change_mean": -0.25470156222581863, + "reward_change_min": -0.4568561315536499, + "reward_change_std": 0.16765617858618498, + "reward_std": 0.6069091446697712, + "rewards/accuracy_reward": 0.1458333358168602, + "rewards/cosine_scaled_reward": -0.09532715613022447, + "step": 463 + }, + { + "clip_fraction": 0.0, + "completion_length": 1749.0416717529297, + "epoch": 0.5302857142857142, + "grad_norm": 0.11516103893518448, + "kl": 3.466010093688965e-05, + "lambda_div_used": 0.5341480076313019, + "learning_rate": 1.1492947512799328e-07, + "loss": -0.0088, + "reward": -0.040389321744441986, + "reward_after_mean": -0.040389321744441986, + "reward_after_std": 0.37696985714137554, + "reward_before_mean": 0.5386080192402005, + "reward_before_std": 0.14989514648914337, + "reward_change_max": 0.0, + "reward_change_mean": -0.5789973307400942, + "reward_change_min": -0.7674467526376247, + "reward_change_std": 0.2917799688875675, + "reward_std": 0.37696986459195614, + "rewards/accuracy_reward": 0.375, + "rewards/cosine_scaled_reward": 0.16360801365226507, + "step": 464 + }, + { + "clip_fraction": 0.0, + "completion_length": 3036.854179382324, + "epoch": 0.5314285714285715, + "grad_norm": 0.08283663541078568, + "kl": 2.4955719709396362e-05, + "lambda_div_used": 0.572467640042305, + "learning_rate": 1.1413757749211602e-07, + "loss": 0.0373, + "reward": -0.40219551732297987, + "reward_after_mean": -0.40219551732297987, + "reward_after_std": 0.40797613374888897, + "reward_before_mean": -0.1455877646803856, + "reward_before_std": 0.32708599977195263, + "reward_change_max": 0.0, + "reward_change_mean": -0.256607748568058, + "reward_change_min": -0.37336331233382225, + "reward_change_std": 0.1377662243321538, + "reward_std": 0.40797614119946957, + "rewards/accuracy_reward": 0.0416666679084301, + "rewards/cosine_scaled_reward": -0.18725443072617054, + "step": 465 + }, + { + "clip_fraction": 0.0, + "completion_length": 3122.9583435058594, + "epoch": 0.5325714285714286, + "grad_norm": 0.07085831463336945, + "kl": 2.0582228899002075e-05, + "lambda_div_used": 0.5319224968552589, + "learning_rate": 1.1336692317580158e-07, + "loss": -0.0563, + "reward": -0.36368466913700104, + "reward_after_mean": -0.36368466913700104, + "reward_after_std": 0.27606455981731415, + "reward_before_mean": 0.005439521744847298, + "reward_before_std": 0.14014938473701477, + "reward_change_max": 0.0, + "reward_change_mean": -0.36912417598068714, + "reward_change_min": -0.5061142668128014, + "reward_change_std": 0.1899967910721898, + "reward_std": 0.2760645691305399, + "rewards/accuracy_reward": 0.125, + "rewards/cosine_scaled_reward": -0.11956048291176558, + "step": 466 + }, + { + "clip_fraction": 0.0, + "completion_length": 3113.2708854675293, + "epoch": 0.5337142857142857, + "grad_norm": 0.07966674119234085, + "kl": 3.802310675382614e-05, + "lambda_div_used": 0.5612561926245689, + "learning_rate": 1.1261754973965422e-07, + "loss": -0.0067, + "reward": -0.3137255348265171, + "reward_after_mean": -0.3137255348265171, + "reward_after_std": 0.39207486622035503, + "reward_before_mean": 0.03320633992552757, + "reward_before_std": 0.2764817178249359, + "reward_change_max": 0.0, + "reward_change_mean": -0.3469318952411413, + "reward_change_min": -0.4828169047832489, + "reward_change_std": 0.18068420328199863, + "reward_std": 0.3920748811215162, + "rewards/accuracy_reward": 0.14583333395421505, + "rewards/cosine_scaled_reward": -0.11262698657810688, + "step": 467 + }, + { + "clip_fraction": 0.0, + "completion_length": 2898.770854949951, + "epoch": 0.5348571428571428, + "grad_norm": 0.07577298581600189, + "kl": 4.957616329193115e-05, + "lambda_div_used": 0.5363663136959076, + "learning_rate": 1.1188949370707787e-07, + "loss": 0.0338, + "reward": -0.5352834053337574, + "reward_after_mean": -0.5352834053337574, + "reward_after_std": 0.22454985231161118, + "reward_before_mean": -0.2775777019560337, + "reward_before_std": 0.15978037798777223, + "reward_change_max": 0.0, + "reward_change_mean": -0.2577057033777237, + "reward_change_min": -0.3789172098040581, + "reward_change_std": 0.14186344109475613, + "reward_std": 0.22454985417425632, + "rewards/accuracy_reward": 0.0, + "rewards/cosine_scaled_reward": -0.27757770381867886, + "step": 468 + }, + { + "clip_fraction": 0.0, + "completion_length": 2854.8750228881836, + "epoch": 0.536, + "grad_norm": 0.09182782471179962, + "kl": 2.0341947674751282e-05, + "lambda_div_used": 0.5550094842910767, + "learning_rate": 1.1118279056249653e-07, + "loss": 0.0301, + "reward": -0.35969678312540054, + "reward_after_mean": -0.35969678312540054, + "reward_after_std": 0.30523936823010445, + "reward_before_mean": -0.04356633126735687, + "reward_before_std": 0.2476841462776065, + "reward_change_max": 0.0, + "reward_change_mean": -0.31613045185804367, + "reward_change_min": -0.45744797959923744, + "reward_change_std": 0.17962745483964682, + "reward_std": 0.3052393738180399, + "rewards/accuracy_reward": 0.0833333358168602, + "rewards/cosine_scaled_reward": -0.12689966335892677, + "step": 469 + }, + { + "clip_fraction": 0.0, + "completion_length": 3333.562530517578, + "epoch": 0.5371428571428571, + "grad_norm": 0.05610484257340431, + "kl": 2.9811635613441467e-05, + "lambda_div_used": 0.5911690816283226, + "learning_rate": 1.1049747474962444e-07, + "loss": 0.0307, + "reward": -0.37843912467360497, + "reward_after_mean": -0.37843912467360497, + "reward_after_std": 0.49228920973837376, + "reward_before_mean": -0.14763083557772916, + "reward_before_std": 0.4128831513226032, + "reward_change_max": 0.0, + "reward_change_mean": -0.230808287858963, + "reward_change_min": -0.3457215949892998, + "reward_change_std": 0.12627399526536465, + "reward_std": 0.49228921718895435, + "rewards/accuracy_reward": 0.06250000186264515, + "rewards/cosine_scaled_reward": -0.21013083728030324, + "step": 470 + }, + { + "clip_fraction": 0.0, + "completion_length": 3032.312515258789, + "epoch": 0.5382857142857143, + "grad_norm": 0.053547170013189316, + "kl": 2.0138919353485107e-05, + "lambda_div_used": 0.5520770102739334, + "learning_rate": 1.0983357966978745e-07, + "loss": -0.0036, + "reward": -0.32707202807068825, + "reward_after_mean": -0.32707202807068825, + "reward_after_std": 0.30774626694619656, + "reward_before_mean": 0.006061417981982231, + "reward_before_std": 0.23228682670742273, + "reward_change_max": 0.0, + "reward_change_mean": -0.3331334535032511, + "reward_change_min": -0.47613633051514626, + "reward_change_std": 0.181988756172359, + "reward_std": 0.3077462762594223, + "rewards/accuracy_reward": 0.1041666716337204, + "rewards/cosine_scaled_reward": -0.09810524806380272, + "step": 471 + }, + { + "clip_fraction": 0.0, + "completion_length": 3115.8333435058594, + "epoch": 0.5394285714285715, + "grad_norm": 0.05183921754360199, + "kl": 1.757591962814331e-05, + "lambda_div_used": 0.6081600487232208, + "learning_rate": 1.0919113768029517e-07, + "loss": 0.0248, + "reward": -0.22974545462056994, + "reward_after_mean": -0.22974545462056994, + "reward_after_std": 0.5618578754365444, + "reward_before_mean": 0.03380160592496395, + "reward_before_std": 0.4960998175665736, + "reward_change_max": 0.0, + "reward_change_mean": -0.2635470647364855, + "reward_change_min": -0.4026007801294327, + "reward_change_std": 0.1525269951671362, + "reward_std": 0.5618579015135765, + "rewards/accuracy_reward": 0.14583333767950535, + "rewards/cosine_scaled_reward": -0.1120317269815132, + "step": 472 + }, + { + "clip_fraction": 0.0, + "completion_length": 3389.7291870117188, + "epoch": 0.5405714285714286, + "grad_norm": 0.05399390682578087, + "kl": 2.3730099201202393e-05, + "lambda_div_used": 0.6023471653461456, + "learning_rate": 1.0857018009286381e-07, + "loss": -0.0081, + "reward": -0.16527403378859162, + "reward_after_mean": -0.16527403378859162, + "reward_after_std": 0.4859522972255945, + "reward_before_mean": 0.15660450607538223, + "reward_before_std": 0.469901567324996, + "reward_change_max": 0.0, + "reward_change_mean": -0.3218785412609577, + "reward_change_min": -0.5562809407711029, + "reward_change_std": 0.21282578073441982, + "reward_std": 0.48595230281352997, + "rewards/accuracy_reward": 0.2083333395421505, + "rewards/cosine_scaled_reward": -0.051728841848671436, + "step": 473 + }, + { + "clip_fraction": 0.0, + "completion_length": 2609.3541946411133, + "epoch": 0.5417142857142857, + "grad_norm": 0.07730484753847122, + "kl": 1.3811513781547546e-05, + "lambda_div_used": 0.5951687097549438, + "learning_rate": 1.0797073717209013e-07, + "loss": 0.085, + "reward": 0.0976928174495697, + "reward_after_mean": 0.0976928174495697, + "reward_after_std": 0.5153173375874758, + "reward_before_mean": 0.5706379320472479, + "reward_before_std": 0.4373670984059572, + "reward_change_max": 0.0, + "reward_change_mean": -0.47294510155916214, + "reward_change_min": -0.6950589865446091, + "reward_change_std": 0.27640796452760696, + "reward_std": 0.515317365527153, + "rewards/accuracy_reward": 0.3958333432674408, + "rewards/cosine_scaled_reward": 0.1748045664280653, + "step": 474 + }, + { + "clip_fraction": 0.0, + "completion_length": 2361.020851135254, + "epoch": 0.5428571428571428, + "grad_norm": 0.08214546740055084, + "kl": 9.525567293167114e-06, + "lambda_div_used": 0.6207183972001076, + "learning_rate": 1.0739283813397639e-07, + "loss": 0.0204, + "reward": 0.0007177861407399178, + "reward_after_mean": 0.0007177861407399178, + "reward_after_std": 0.5817896965891123, + "reward_before_mean": 0.34289272502064705, + "reward_before_std": 0.5596933793276548, + "reward_change_max": 0.0, + "reward_change_mean": -0.3421749472618103, + "reward_change_min": -0.5431834943592548, + "reward_change_std": 0.21874003671109676, + "reward_std": 0.581789730116725, + "rewards/accuracy_reward": 0.29166667722165585, + "rewards/cosine_scaled_reward": 0.05122605012729764, + "step": 475 + }, + { + "clip_fraction": 0.0, + "completion_length": 2583.8333740234375, + "epoch": 0.544, + "grad_norm": 0.08052952587604523, + "kl": 3.0007213354110718e-05, + "lambda_div_used": 0.6784488782286644, + "learning_rate": 1.068365111445064e-07, + "loss": 0.0154, + "reward": -0.031517775263637304, + "reward_after_mean": -0.031517775263637304, + "reward_after_std": 0.8641770519316196, + "reward_before_mean": 0.2038349723443389, + "reward_before_std": 0.8286011293530464, + "reward_change_max": 0.0, + "reward_change_mean": -0.23535274900496006, + "reward_change_min": -0.41804002970457077, + "reward_change_std": 0.16118939872831106, + "reward_std": 0.8641770891845226, + "rewards/accuracy_reward": 0.22916666977107525, + "rewards/cosine_scaled_reward": -0.025331700686365366, + "step": 476 + }, + { + "clip_fraction": 0.0, + "completion_length": 2308.291732788086, + "epoch": 0.5451428571428572, + "grad_norm": 0.06839341670274734, + "kl": 8.73953104019165e-06, + "lambda_div_used": 0.6096622571349144, + "learning_rate": 1.063017833182728e-07, + "loss": 0.1111, + "reward": 0.19456686079502106, + "reward_after_mean": 0.19456686079502106, + "reward_after_std": 0.5847878716886044, + "reward_before_mean": 0.678333050571382, + "reward_before_std": 0.5006472393870354, + "reward_change_max": 0.0, + "reward_change_mean": -0.48376619443297386, + "reward_change_min": -0.6835142783820629, + "reward_change_std": 0.2774948738515377, + "reward_std": 0.5847878959029913, + "rewards/accuracy_reward": 0.45833334885537624, + "rewards/cosine_scaled_reward": 0.21999971382319927, + "step": 477 + }, + { + "clip_fraction": 0.0, + "completion_length": 3169.3125610351562, + "epoch": 0.5462857142857143, + "grad_norm": 0.052864328026771545, + "kl": 6.990041583776474e-06, + "lambda_div_used": 0.5758631080389023, + "learning_rate": 1.0578868071715544e-07, + "loss": 0.0349, + "reward": -0.23655124567449093, + "reward_after_mean": -0.23655124567449093, + "reward_after_std": 0.4532997701317072, + "reward_before_mean": 0.11144567281007767, + "reward_before_std": 0.3459607223048806, + "reward_change_max": 0.0, + "reward_change_mean": -0.3479969333857298, + "reward_change_min": -0.5187661461532116, + "reward_change_std": 0.19546091184020042, + "reward_std": 0.4532997887581587, + "rewards/accuracy_reward": 0.20833333395421505, + "rewards/cosine_scaled_reward": -0.09688764810562134, + "step": 478 + }, + { + "clip_fraction": 0.0, + "completion_length": 2825.916702270508, + "epoch": 0.5474285714285714, + "grad_norm": 0.08355361968278885, + "kl": 3.739655949175358e-05, + "lambda_div_used": 0.5902442187070847, + "learning_rate": 1.0529722834905125e-07, + "loss": 0.033, + "reward": -0.37585191056132317, + "reward_after_mean": -0.37585191056132317, + "reward_after_std": 0.45466959848999977, + "reward_before_mean": -0.12638765759766102, + "reward_before_std": 0.4168459586799145, + "reward_change_max": 0.0, + "reward_change_mean": -0.2494642548263073, + "reward_change_min": -0.4229847304522991, + "reward_change_std": 0.15689124166965485, + "reward_std": 0.4546696189790964, + "rewards/accuracy_reward": 0.06250000186264515, + "rewards/cosine_scaled_reward": -0.18888765573501587, + "step": 479 + }, + { + "clip_fraction": 0.0, + "completion_length": 2415.895866394043, + "epoch": 0.5485714285714286, + "grad_norm": 0.08123134076595306, + "kl": 1.4044344425201416e-05, + "lambda_div_used": 0.5890126600861549, + "learning_rate": 1.0482745016665526e-07, + "loss": 0.0739, + "reward": -0.36289478465914726, + "reward_after_mean": -0.36289478465914726, + "reward_after_std": 0.4815117195248604, + "reward_before_mean": -0.11086839716881514, + "reward_before_std": 0.4026770405471325, + "reward_change_max": 0.0, + "reward_change_mean": -0.25202639400959015, + "reward_change_min": -0.37619682028889656, + "reward_change_std": 0.13623447716236115, + "reward_std": 0.4815117232501507, + "rewards/accuracy_reward": 0.06250000186264515, + "rewards/cosine_scaled_reward": -0.17336839414201677, + "step": 480 + }, + { + "clip_fraction": 0.0, + "completion_length": 3057.625030517578, + "epoch": 0.5497142857142857, + "grad_norm": 0.05632660165429115, + "kl": 2.8233975172042847e-05, + "lambda_div_used": 0.5841230228543282, + "learning_rate": 1.0437936906629334e-07, + "loss": 0.0198, + "reward": -0.386552257463336, + "reward_after_mean": -0.386552257463336, + "reward_after_std": 0.4468122199177742, + "reward_before_mean": -0.14256569184362888, + "reward_before_std": 0.38717135414481163, + "reward_change_max": 0.0, + "reward_change_mean": -0.24398657120764256, + "reward_change_min": -0.42186837270855904, + "reward_change_std": 0.14736179821193218, + "reward_std": 0.4468122236430645, + "rewards/accuracy_reward": 0.06250000186264515, + "rewards/cosine_scaled_reward": -0.20506569370627403, + "step": 481 + }, + { + "clip_fraction": 0.0, + "completion_length": 2824.5833587646484, + "epoch": 0.5508571428571428, + "grad_norm": 0.0865793228149414, + "kl": 2.4417764507234097e-05, + "lambda_div_used": 0.622517004609108, + "learning_rate": 1.0395300688680625e-07, + "loss": -0.0313, + "reward": -0.05876433290541172, + "reward_after_mean": -0.05876433290541172, + "reward_after_std": 0.5989817604422569, + "reward_before_mean": 0.259432727470994, + "reward_before_std": 0.5737235806882381, + "reward_change_max": 0.0, + "reward_change_mean": -0.31819707341492176, + "reward_change_min": -0.5269695781171322, + "reward_change_std": 0.21138717606663704, + "reward_std": 0.5989817790687084, + "rewards/accuracy_reward": 0.22916667349636555, + "rewards/cosine_scaled_reward": 0.030266055837273598, + "step": 482 + }, + { + "clip_fraction": 0.0, + "completion_length": 2869.250015258789, + "epoch": 0.552, + "grad_norm": 0.08511485904455185, + "kl": 3.474205732345581e-05, + "lambda_div_used": 0.6013401597738266, + "learning_rate": 1.0354838440848501e-07, + "loss": -0.0837, + "reward": -0.2791392467916012, + "reward_after_mean": -0.2791392467916012, + "reward_after_std": 0.5272991992533207, + "reward_before_mean": -0.02371177263557911, + "reward_before_std": 0.4689871799200773, + "reward_change_max": 0.0, + "reward_change_mean": -0.2554274704307318, + "reward_change_min": -0.40494704991579056, + "reward_change_std": 0.15485912468284369, + "reward_std": 0.5272992141544819, + "rewards/accuracy_reward": 0.1041666679084301, + "rewards/cosine_scaled_reward": -0.1278784405440092, + "step": 483 + }, + { + "clip_fraction": 0.0, + "completion_length": 2591.7500228881836, + "epoch": 0.5531428571428572, + "grad_norm": 0.08745326101779938, + "kl": 3.6368146538734436e-07, + "lambda_div_used": 0.6466837078332901, + "learning_rate": 1.0316552135205837e-07, + "loss": 0.0082, + "reward": -0.04187892563641071, + "reward_after_mean": -0.04187892563641071, + "reward_after_std": 0.6843612100929022, + "reward_before_mean": 0.242530676885508, + "reward_before_std": 0.6844684220850468, + "reward_change_max": 0.0, + "reward_change_mean": -0.284409599378705, + "reward_change_min": -0.49204646795988083, + "reward_change_std": 0.1993796620517969, + "reward_std": 0.6843612212687731, + "rewards/accuracy_reward": 0.2708333395421505, + "rewards/cosine_scaled_reward": -0.0283026653341949, + "step": 484 + }, + { + "clip_fraction": 0.0, + "completion_length": 2046.2500381469727, + "epoch": 0.5542857142857143, + "grad_norm": 0.09076947718858719, + "kl": 2.752244472503662e-05, + "lambda_div_used": 0.5719796344637871, + "learning_rate": 1.0280443637773163e-07, + "loss": 0.0035, + "reward": -0.40644849208183587, + "reward_after_mean": -0.40644849208183587, + "reward_after_std": 0.398155614733696, + "reward_before_mean": -0.14403727410535794, + "reward_before_std": 0.3282261691056192, + "reward_change_max": 0.0, + "reward_change_mean": -0.2624112106859684, + "reward_change_min": -0.43003450334072113, + "reward_change_std": 0.15388297475874424, + "reward_std": 0.39815562404692173, + "rewards/accuracy_reward": 0.06250000186264515, + "rewards/cosine_scaled_reward": -0.20653727487660944, + "step": 485 + }, + { + "clip_fraction": 0.0, + "completion_length": 1967.7916946411133, + "epoch": 0.5554285714285714, + "grad_norm": 0.09958811849355698, + "kl": 3.91155481338501e-05, + "lambda_div_used": 0.5703976079821587, + "learning_rate": 1.0246514708427701e-07, + "loss": 0.0966, + "reward": -0.26609380822628736, + "reward_after_mean": -0.26609380822628736, + "reward_after_std": 0.4386676363646984, + "reward_before_mean": 0.07946969009935856, + "reward_before_std": 0.31633214373141527, + "reward_change_max": 0.0, + "reward_change_mean": -0.34556350111961365, + "reward_change_min": -0.48576217144727707, + "reward_change_std": 0.1828850321471691, + "reward_std": 0.4386676475405693, + "rewards/accuracy_reward": 0.1666666679084301, + "rewards/cosine_scaled_reward": -0.08719698037020862, + "step": 486 + }, + { + "clip_fraction": 0.0, + "completion_length": 1967.0625228881836, + "epoch": 0.5565714285714286, + "grad_norm": 0.10718576610088348, + "kl": 3.7085264921188354e-06, + "lambda_div_used": 0.6087248548865318, + "learning_rate": 1.0214767000817596e-07, + "loss": 0.0461, + "reward": 0.08780635055154562, + "reward_after_mean": 0.08780635055154562, + "reward_after_std": 0.6550032701343298, + "reward_before_mean": 0.5625860100844875, + "reward_before_std": 0.5011934200301766, + "reward_change_max": 0.0, + "reward_change_mean": -0.47477963753044605, + "reward_change_min": -0.7034505233168602, + "reward_change_std": 0.26853484753519297, + "reward_std": 0.6550032943487167, + "rewards/accuracy_reward": 0.3958333358168602, + "rewards/cosine_scaled_reward": 0.16675265738740563, + "step": 487 + }, + { + "clip_fraction": 0.0, + "completion_length": 2356.8125228881836, + "epoch": 0.5577142857142857, + "grad_norm": 0.08583448082208633, + "kl": 2.1670013666152954e-05, + "lambda_div_used": 0.5598724335432053, + "learning_rate": 1.0185202062281336e-07, + "loss": -0.0446, + "reward": -0.39712974801659584, + "reward_after_mean": -0.39712974801659584, + "reward_after_std": 0.3282298669219017, + "reward_before_mean": -0.11880321707576513, + "reward_before_std": 0.2731999019160867, + "reward_change_max": 0.0, + "reward_change_mean": -0.27832652255892754, + "reward_change_min": -0.42709672823548317, + "reward_change_std": 0.16301770228892565, + "reward_std": 0.3282298743724823, + "rewards/accuracy_reward": 0.0833333358168602, + "rewards/cosine_scaled_reward": -0.20213654916733503, + "step": 488 + }, + { + "clip_fraction": 0.0, + "completion_length": 3039.2916870117188, + "epoch": 0.5588571428571428, + "grad_norm": 0.08493036031723022, + "kl": 4.955753684043884e-05, + "lambda_div_used": 0.5549413114786148, + "learning_rate": 1.0157821333772304e-07, + "loss": -0.0127, + "reward": -0.4395306259393692, + "reward_after_mean": -0.4395306259393692, + "reward_after_std": 0.30519232526421547, + "reward_before_mean": -0.16155868768692017, + "reward_before_std": 0.24803727120161057, + "reward_change_max": 0.0, + "reward_change_mean": -0.27797193080186844, + "reward_change_min": -0.44194458797574043, + "reward_change_std": 0.1623495165258646, + "reward_std": 0.3051923308521509, + "rewards/accuracy_reward": 0.0625, + "rewards/cosine_scaled_reward": -0.22405867651104927, + "step": 489 + }, + { + "clip_fraction": 0.0, + "completion_length": 2838.0833892822266, + "epoch": 0.56, + "grad_norm": 0.06069381162524223, + "kl": 1.2032687664031982e-05, + "lambda_div_used": 0.583185076713562, + "learning_rate": 1.013262614978859e-07, + "loss": 0.0139, + "reward": -0.1338807214051485, + "reward_after_mean": -0.1338807214051485, + "reward_after_std": 0.4649778436869383, + "reward_before_mean": 0.25663699954748154, + "reward_before_std": 0.3789667785167694, + "reward_change_max": 0.0, + "reward_change_mean": -0.39051773957908154, + "reward_change_min": -0.6341840960085392, + "reward_change_std": 0.2300328817218542, + "reward_std": 0.46497784554958344, + "rewards/accuracy_reward": 0.2708333395421505, + "rewards/cosine_scaled_reward": -0.014196328818798065, + "step": 490 + }, + { + "clip_fraction": 0.0, + "completion_length": 2593.7291870117188, + "epoch": 0.5611428571428572, + "grad_norm": 0.07340901345014572, + "kl": 3.531062975525856e-05, + "lambda_div_used": 0.5965912714600563, + "learning_rate": 1.0109617738307911e-07, + "loss": 0.0268, + "reward": 0.01746355928480625, + "reward_after_mean": 0.01746355928480625, + "reward_after_std": 0.5940692499279976, + "reward_before_mean": 0.47233812790364027, + "reward_before_std": 0.4421461224555969, + "reward_change_max": 0.0, + "reward_change_mean": -0.4548745583742857, + "reward_change_min": -0.6400704979896545, + "reward_change_std": 0.24487750884145498, + "reward_std": 0.5940692741423845, + "rewards/accuracy_reward": 0.3541666679084301, + "rewards/cosine_scaled_reward": 0.11817142926156521, + "step": 491 + }, + { + "clip_fraction": 0.0, + "completion_length": 2583.5416679382324, + "epoch": 0.5622857142857143, + "grad_norm": 0.08282249420881271, + "kl": 2.4866312742233276e-05, + "lambda_div_used": 0.5367531701922417, + "learning_rate": 1.0088797220727779e-07, + "loss": 0.0376, + "reward": -0.3908469006419182, + "reward_after_mean": -0.3908469006419182, + "reward_after_std": 0.27869052439928055, + "reward_before_mean": -0.03699151985347271, + "reward_before_std": 0.16138391755521297, + "reward_change_max": 0.0, + "reward_change_mean": -0.3538553789258003, + "reward_change_min": -0.5079905577003956, + "reward_change_std": 0.18725439626723528, + "reward_std": 0.27869053184986115, + "rewards/accuracy_reward": 0.125, + "rewards/cosine_scaled_reward": -0.1619915273040533, + "step": 492 + }, + { + "clip_fraction": 0.0, + "completion_length": 2176.8333740234375, + "epoch": 0.5634285714285714, + "grad_norm": 0.09936831891536713, + "kl": 6.021931767463684e-06, + "lambda_div_used": 0.596942737698555, + "learning_rate": 1.0070165611810855e-07, + "loss": 0.0246, + "reward": -0.037120603024959564, + "reward_after_mean": -0.037120603024959564, + "reward_after_std": 0.46812310442328453, + "reward_before_mean": 0.34137603268027306, + "reward_before_std": 0.44823691714555025, + "reward_change_max": 0.0, + "reward_change_mean": -0.3784966245293617, + "reward_change_min": -0.5661123469471931, + "reward_change_std": 0.234741548076272, + "reward_std": 0.4681231305003166, + "rewards/accuracy_reward": 0.2708333432674408, + "rewards/cosine_scaled_reward": 0.07054269965738058, + "step": 493 + }, + { + "clip_fraction": 0.0, + "completion_length": 2352.583427429199, + "epoch": 0.5645714285714286, + "grad_norm": 0.07574566453695297, + "kl": 1.6648322343826294e-05, + "lambda_div_used": 0.6796721071004868, + "learning_rate": 1.005372381963547e-07, + "loss": 0.092, + "reward": 0.10963849350810051, + "reward_after_mean": 0.10963849350810051, + "reward_after_std": 0.8239130303263664, + "reward_before_mean": 0.4074633736163378, + "reward_before_std": 0.848267612978816, + "reward_change_max": 0.0, + "reward_change_mean": -0.2978248745203018, + "reward_change_min": -0.6511921100318432, + "reward_change_std": 0.2445714958012104, + "reward_std": 0.8239130582660437, + "rewards/accuracy_reward": 0.3125000037252903, + "rewards/cosine_scaled_reward": 0.09496336756274104, + "step": 494 + }, + { + "clip_fraction": 0.0, + "completion_length": 3203.8959045410156, + "epoch": 0.5657142857142857, + "grad_norm": 0.057527996599674225, + "kl": 2.3846514523029327e-05, + "lambda_div_used": 0.5994853675365448, + "learning_rate": 1.0039472645551372e-07, + "loss": 0.0637, + "reward": -0.21300244145095348, + "reward_after_mean": -0.21300244145095348, + "reward_after_std": 0.48377062380313873, + "reward_before_mean": 0.08397414721548557, + "reward_before_std": 0.4531490486115217, + "reward_change_max": 0.0, + "reward_change_mean": -0.29697658494114876, + "reward_change_min": -0.49157723411917686, + "reward_change_std": 0.1875515878200531, + "reward_std": 0.4837706368416548, + "rewards/accuracy_reward": 0.16666667349636555, + "rewards/cosine_scaled_reward": -0.08269252139143646, + "step": 495 + }, + { + "clip_fraction": 0.0, + "completion_length": 1895.9375457763672, + "epoch": 0.5668571428571428, + "grad_norm": 0.10791665315628052, + "kl": 1.890142448246479e-05, + "lambda_div_used": 0.6198414713144302, + "learning_rate": 1.002741278414069e-07, + "loss": 0.0183, + "reward": 0.09977892541792244, + "reward_after_mean": 0.09977892541792244, + "reward_after_std": 0.6343465056270361, + "reward_before_mean": 0.5255357641726732, + "reward_before_std": 0.555769513361156, + "reward_change_max": 0.0, + "reward_change_mean": -0.4257568307220936, + "reward_change_min": -0.6588771641254425, + "reward_change_std": 0.25742017664015293, + "reward_std": 0.634346516802907, + "rewards/accuracy_reward": 0.35416667349636555, + "rewards/cosine_scaled_reward": 0.17136909160763025, + "step": 496 + }, + { + "clip_fraction": 0.0, + "completion_length": 2577.645851135254, + "epoch": 0.568, + "grad_norm": 0.06887742131948471, + "kl": 2.493336796760559e-05, + "lambda_div_used": 0.6232517957687378, + "learning_rate": 1.0017544823184055e-07, + "loss": 0.0143, + "reward": 0.039356768131256104, + "reward_after_mean": 0.039356768131256104, + "reward_after_std": 0.5658366903662682, + "reward_before_mean": 0.401943476870656, + "reward_before_std": 0.5692849718034267, + "reward_change_max": 0.0, + "reward_change_mean": -0.362586697563529, + "reward_change_min": -0.6212925836443901, + "reward_change_std": 0.24119753576815128, + "reward_std": 0.5658366959542036, + "rewards/accuracy_reward": 0.3333333469927311, + "rewards/cosine_scaled_reward": 0.06861014291644096, + "step": 497 + }, + { + "clip_fraction": 0.0, + "completion_length": 2857.2916946411133, + "epoch": 0.5691428571428572, + "grad_norm": 0.06479962170124054, + "kl": 9.963754564523697e-06, + "lambda_div_used": 0.6236685812473297, + "learning_rate": 1.0009869243631952e-07, + "loss": 0.0008, + "reward": -0.18094617873430252, + "reward_after_mean": -0.18094617873430252, + "reward_after_std": 0.6155845355242491, + "reward_before_mean": 0.07468715589493513, + "reward_before_std": 0.5768492119386792, + "reward_change_max": 0.0, + "reward_change_mean": -0.2556333262473345, + "reward_change_min": -0.4431779384613037, + "reward_change_std": 0.16846577636897564, + "reward_std": 0.6155845616012812, + "rewards/accuracy_reward": 0.1666666679084301, + "rewards/cosine_scaled_reward": -0.0919795110821724, + "step": 498 + }, + { + "clip_fraction": 0.0, + "completion_length": 2707.791702270508, + "epoch": 0.5702857142857143, + "grad_norm": 0.06317302584648132, + "kl": 1.5532365068793297e-05, + "lambda_div_used": 0.6250721588730812, + "learning_rate": 1.000438641958131e-07, + "loss": -0.0298, + "reward": -0.12677906453609467, + "reward_after_mean": -0.12677906453609467, + "reward_after_std": 0.6044176463037729, + "reward_before_mean": 0.1698420336470008, + "reward_before_std": 0.5829241154715419, + "reward_change_max": 0.0, + "reward_change_mean": -0.2966211009770632, + "reward_change_min": -0.5905468240380287, + "reward_change_std": 0.2108100038021803, + "reward_std": 0.6044176481664181, + "rewards/accuracy_reward": 0.2083333358168602, + "rewards/cosine_scaled_reward": -0.03849130589514971, + "step": 499 + }, + { + "clip_fraction": 0.0, + "completion_length": 2980.9167098999023, + "epoch": 0.5714285714285714, + "grad_norm": 0.0621151477098465, + "kl": 1.033581793308258e-05, + "lambda_div_used": 0.6023986041545868, + "learning_rate": 1.0001096618257236e-07, + "loss": 0.0314, + "reward": -0.26747864981007297, + "reward_after_mean": -0.26747864981007297, + "reward_after_std": 0.5251844674348831, + "reward_before_mean": -0.0025623496621847153, + "reward_before_std": 0.4696931503713131, + "reward_change_max": 0.0, + "reward_change_mean": -0.26491627655923367, + "reward_change_min": -0.43205036222934723, + "reward_change_std": 0.15884541906416416, + "reward_std": 0.5251844730228186, + "rewards/accuracy_reward": 0.1041666679084301, + "rewards/cosine_scaled_reward": -0.10672901570796967, + "step": 500 + }, + { + "epoch": 0.5714285714285714, + "step": 500, + "total_flos": 0.0, + "train_loss": 0.005314271912367986, + "train_runtime": 106074.3369, + "train_samples_per_second": 0.226, + "train_steps_per_second": 0.005 + } + ], + "logging_steps": 1, + "max_steps": 500, + "num_input_tokens_seen": 0, + "num_train_epochs": 1, + "save_steps": 10, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": true + }, + "attributes": {} + } + }, + "total_flos": 0.0, + "train_batch_size": 6, + "trial_name": null, + "trial_params": null +} diff --git a/training_args.bin b/training_args.bin new file mode 100644 index 0000000..ca8fae2 --- /dev/null +++ b/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:98a2484bdddb222de3c5a9d658c6bc6ca9f8040b634d892d790007ff9f28adbc +size 8824