commit 70ec67fed4130f9eec8c27df7c2299184ef9dabe Author: ModelHub XC Date: Sat Jun 13 17:02:19 2026 +0800 初始化项目，由ModelHub XC社区提供模型 Model: kikiyaa/qwen-dpo-finetuned-ver2 Source: Original Platform diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..69797b7 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,39 @@ +*.7z filter=lfs diff=lfs merge=lfs -text +*.arrow filter=lfs diff=lfs merge=lfs -text +*.bin filter=lfs diff=lfs merge=lfs -text +*.bz2 filter=lfs diff=lfs merge=lfs -text +*.ckpt filter=lfs diff=lfs merge=lfs -text +*.ftz filter=lfs diff=lfs merge=lfs -text +*.gz filter=lfs diff=lfs merge=lfs -text +*.h5 filter=lfs diff=lfs merge=lfs -text +*.joblib filter=lfs diff=lfs merge=lfs -text +*.lfs.* filter=lfs diff=lfs merge=lfs -text +*.mlmodel filter=lfs diff=lfs merge=lfs -text +*.model filter=lfs diff=lfs merge=lfs -text +*.msgpack filter=lfs diff=lfs merge=lfs -text +*.npy filter=lfs diff=lfs merge=lfs -text +*.npz filter=lfs diff=lfs merge=lfs -text +*.onnx filter=lfs diff=lfs merge=lfs -text +*.ot filter=lfs diff=lfs merge=lfs -text +*.parquet filter=lfs diff=lfs merge=lfs -text +*.pb filter=lfs diff=lfs merge=lfs -text +*.pickle filter=lfs diff=lfs merge=lfs -text +*.pkl filter=lfs diff=lfs merge=lfs -text +*.pt filter=lfs diff=lfs merge=lfs -text +*.pth filter=lfs diff=lfs merge=lfs -text +*.rar filter=lfs diff=lfs merge=lfs -text +*.safetensors filter=lfs diff=lfs merge=lfs -text +saved_model/**/* filter=lfs diff=lfs merge=lfs -text +*.tar.* filter=lfs diff=lfs merge=lfs -text +*.tar filter=lfs diff=lfs merge=lfs -text +*.tflite filter=lfs diff=lfs merge=lfs -text +*.tgz filter=lfs diff=lfs merge=lfs -text +*.wasm filter=lfs diff=lfs merge=lfs -text +*.xz filter=lfs diff=lfs merge=lfs -text +*.zip filter=lfs diff=lfs merge=lfs -text +*.zst filter=lfs diff=lfs merge=lfs -text +*tfevents* filter=lfs diff=lfs merge=lfs -text +checkpoint-1000/tokenizer.json filter=lfs diff=lfs merge=lfs -text +checkpoint-1013/tokenizer.json filter=lfs diff=lfs merge=lfs -text +checkpoint-900/tokenizer.json filter=lfs diff=lfs merge=lfs -text +tokenizer.json filter=lfs diff=lfs merge=lfs -text diff --git a/README.md b/README.md new file mode 100644 index 0000000..2b3ea7b --- /dev/null +++ b/README.md @@ -0,0 +1,69 @@ +--- +base_model: Qwen/Qwen2.5-7B +library_name: transformers +model_name: kikiyaa/qwen-dpo-finetuned-ver2 +tags: +- generated_from_trainer +- dpo +- trl +licence: license +--- + +# Model Card for kikiyaa/qwen-dpo-finetuned-ver2 + +This model is a fine-tuned version of [Qwen/Qwen2.5-7B](https://huggingface.co/Qwen/Qwen2.5-7B). +It has been trained using [TRL](https://github.com/huggingface/trl). + +## Quick start + +```python +from transformers import pipeline + +question = "If you had a time machine, but could only go to the past or the future once and never return, which would you choose and why?" +generator = pipeline("text-generation", model="None", device="cuda") +output = generator([{"role": "user", "content": question}], max_new_tokens=128, return_full_text=False)[0] +print(output["generated_text"]) +``` + +## Training procedure + + + + + +This model was trained with DPO, a method introduced in [Direct Preference Optimization: Your Language Model is Secretly a Reward Model](https://huggingface.co/papers/2305.18290). + +### Framework versions + +- TRL: 1.1.0 +- Transformers: 5.5.4 +- Pytorch: 2.9.1+cu128 +- Datasets: 4.8.4 +- Tokenizers: 0.22.2 + +## Citations + +Cite DPO as: + +```bibtex +@inproceedings{rafailov2023direct, + title = {{Direct Preference Optimization: Your Language Model is Secretly a Reward Model}}, + author = {Rafael Rafailov and Archit Sharma and Eric Mitchell and Christopher D. Manning and Stefano Ermon and Chelsea Finn}, + year = 2023, + booktitle = {Advances in Neural Information Processing Systems 36: Annual Conference on Neural Information Processing Systems 2023, NeurIPS 2023, New Orleans, LA, USA, December 10 - 16, 2023}, + url = {http://papers.nips.cc/paper_files/paper/2023/hash/a85b405ed65c6477a4fe8302b5e06ce7-Abstract-Conference.html}, + editor = {Alice Oh and Tristan Naumann and Amir Globerson and Kate Saenko and Moritz Hardt and Sergey Levine}, +} +``` + +Cite TRL as: + +```bibtex +@software{vonwerra2020trl, + title = {{TRL: Transformers Reinforcement Learning}}, + author = {von Werra, Leandro and Belkada, Younes and Tunstall, Lewis and Beeching, Edward and Thrush, Tristan and Lambert, Nathan and Huang, Shengyi and Rasul, Kashif and Gallouédec, Quentin}, + license = {Apache-2.0}, + url = {https://github.com/huggingface/trl}, + year = {2020} +} +``` \ No newline at end of file diff --git a/chat_template.jinja b/chat_template.jinja new file mode 100644 index 0000000..28028c0 --- /dev/null +++ b/chat_template.jinja @@ -0,0 +1,54 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0]['role'] == 'system' %} + {{- messages[0]['content'] }} + {%- else %} + {{- 'You are a helpful assistant.' }} + {%- endif %} + {{- "\n\n# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0]['role'] == 'system' %} + {{- '<|im_start|>system\n' + messages[0]['content'] + '<|im_end|>\n' }} + {%- else %} + {{- '<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- for message in messages %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) or (message.role == "assistant" and not message.tool_calls) %} + {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {{- '<|im_start|>' + message.role }} + {%- if message.content %} + {{- '\n' + message.content }} + {%- endif %} + {%- for tool_call in message.tool_calls %} + {%- if tool_call.function is defined %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {{- tool_call.arguments | tojson }} + {{- '}\n' }} + {%- endfor %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- message.content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} +{%- endif %} diff --git a/checkpoint-1000/chat_template.jinja b/checkpoint-1000/chat_template.jinja new file mode 100644 index 0000000..28028c0 --- /dev/null +++ b/checkpoint-1000/chat_template.jinja @@ -0,0 +1,54 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0]['role'] == 'system' %} + {{- messages[0]['content'] }} + {%- else %} + {{- 'You are a helpful assistant.' }} + {%- endif %} + {{- "\n\n# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0]['role'] == 'system' %} + {{- '<|im_start|>system\n' + messages[0]['content'] + '<|im_end|>\n' }} + {%- else %} + {{- '<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- for message in messages %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) or (message.role == "assistant" and not message.tool_calls) %} + {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {{- '<|im_start|>' + message.role }} + {%- if message.content %} + {{- '\n' + message.content }} + {%- endif %} + {%- for tool_call in message.tool_calls %} + {%- if tool_call.function is defined %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {{- tool_call.arguments | tojson }} + {{- '}\n' }} + {%- endfor %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- message.content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} +{%- endif %} diff --git a/checkpoint-1000/config.json b/checkpoint-1000/config.json new file mode 100644 index 0000000..6b475aa --- /dev/null +++ b/checkpoint-1000/config.json @@ -0,0 +1,62 @@ +{ + "architectures": [ + "Qwen2ForCausalLM" + ], + "attention_dropout": 0.0, + "bos_token_id": null, + "dtype": "bfloat16", + "eos_token_id": 151643, + "hidden_act": "silu", + "hidden_size": 3584, + "initializer_range": 0.02, + "intermediate_size": 18944, + "layer_types": [ + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention" + ], + "max_position_embeddings": 131072, + "max_window_layers": 28, + "model_type": "qwen2", + "num_attention_heads": 28, + "num_hidden_layers": 28, + "num_key_value_heads": 4, + "pad_token_id": 151643, + "rms_norm_eps": 1e-06, + "rope_parameters": { + "rope_theta": 1000000.0, + "rope_type": "default" + }, + "sliding_window": null, + "tie_word_embeddings": false, + "transformers_version": "5.5.4", + "use_cache": false, + "use_mrope": false, + "use_sliding_window": false, + "vocab_size": 152064 +} diff --git a/checkpoint-1000/generation_config.json b/checkpoint-1000/generation_config.json new file mode 100644 index 0000000..0fb69a0 --- /dev/null +++ b/checkpoint-1000/generation_config.json @@ -0,0 +1,9 @@ +{ + "do_sample": true, + "eos_token_id": [ + 151643 + ], + "max_new_tokens": 2048, + "pad_token_id": 151643, + "transformers_version": "5.5.4" +} diff --git a/checkpoint-1000/model.safetensors b/checkpoint-1000/model.safetensors new file mode 100644 index 0000000..38a0686 --- /dev/null +++ b/checkpoint-1000/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e28819cb95dc7051e14b3b57b19f88c5ea6651903a4f6a5ebacf2018e3b293ea +size 15231272152 diff --git a/checkpoint-1000/optimizer.pt b/checkpoint-1000/optimizer.pt new file mode 100644 index 0000000..f2402d4 --- /dev/null +++ b/checkpoint-1000/optimizer.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0cbbb6cbe04ad48af3847f35292f569cd8a8fc7ef354f2da0ade0cebe6a1b0a9 +size 18724493813 diff --git a/checkpoint-1000/rng_state.pth b/checkpoint-1000/rng_state.pth new file mode 100644 index 0000000..8791954 --- /dev/null +++ b/checkpoint-1000/rng_state.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:20ea3a198ff666cb4ace1c684b598fe43fc7c3c276b83efc553a1b787e12a304 +size 14645 diff --git a/checkpoint-1000/scheduler.pt b/checkpoint-1000/scheduler.pt new file mode 100644 index 0000000..1eba129 --- /dev/null +++ b/checkpoint-1000/scheduler.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c56152fa88730a7e1a776c7f74a5ea346160275aa1b5687ce5dd5125bd42c29c +size 1465 diff --git a/checkpoint-1000/tokenizer.json b/checkpoint-1000/tokenizer.json new file mode 100644 index 0000000..34510ff --- /dev/null +++ b/checkpoint-1000/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3fd169731d2cbde95e10bf356d66d5997fd885dd8dbb6fb4684da3f23b2585d8 +size 11421892 diff --git a/checkpoint-1000/tokenizer_config.json b/checkpoint-1000/tokenizer_config.json new file mode 100644 index 0000000..c960ecf --- /dev/null +++ b/checkpoint-1000/tokenizer_config.json @@ -0,0 +1,29 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/checkpoint-1000/trainer_state.json b/checkpoint-1000/trainer_state.json new file mode 100644 index 0000000..4203f68 --- /dev/null +++ b/checkpoint-1000/trainer_state.json @@ -0,0 +1,2061 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 2.96, + "eval_steps": 100, + "global_step": 1000, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0, + "eval_entropy": 1.7289148477407603, + "eval_logits/chosen": 0.8636333912138532, + "eval_logits/rejected": -0.6579137279251289, + "eval_logps/chosen": -111.15125904216633, + "eval_logps/rejected": -175.04988199514108, + "eval_loss": 0.6931471228599548, + "eval_mean_token_accuracy": 0.4934375546612106, + "eval_num_tokens": 0.0, + "eval_rewards/accuracies": 0.0, + "eval_rewards/chosen": 0.0, + "eval_rewards/margins": 0.0, + "eval_rewards/rejected": 0.0, + "eval_runtime": 39.0805, + "eval_samples_per_second": 7.293, + "eval_steps_per_second": 3.659, + "step": 0 + }, + { + "entropy": 1.7504533976316452, + "epoch": 0.002962962962962963, + "grad_norm": 83.5, + "learning_rate": 0.0, + "logits/chosen": 1.08207520218349, + "logits/rejected": -0.5940652974601717, + "logps/chosen": -110.63330745697021, + "logps/rejected": -182.49396514892578, + "loss": 0.6931471824645996, + "mean_token_accuracy": 0.5048413500189781, + "num_tokens": 10169.0, + "rewards/accuracies": 0.0, + "rewards/chosen": 0.0, + "rewards/margins": 0.0, + "rewards/rejected": 0.0, + "step": 1 + }, + { + "entropy": 1.7299097461832895, + "epoch": 0.02962962962962963, + "grad_norm": 29.875, + "learning_rate": 4.455445544554456e-06, + "logits/chosen": 0.8212555533910425, + "logits/rejected": -0.6693587809238846, + "logps/chosen": -100.81309244367812, + "logps/rejected": -171.8307899898953, + "loss": 0.5316412713792589, + "mean_token_accuracy": 0.5100594382319186, + "num_tokens": 89122.0, + "rewards/accuracies": 0.8125, + "rewards/chosen": 0.306247040655257, + "rewards/margins": 0.41913801974280634, + "rewards/rejected": -0.11289098787690616, + "step": 10 + }, + { + "entropy": 1.5854810774326324, + "epoch": 0.05925925925925926, + "grad_norm": 0.07470703125, + "learning_rate": 9.405940594059407e-06, + "logits/chosen": 1.2175614897456257, + "logits/rejected": -0.8402934758508309, + "logps/chosen": -77.16593537330627, + "logps/rejected": -196.31047840118407, + "loss": 0.036623942852020266, + "mean_token_accuracy": 0.5996901635080576, + "num_tokens": 182773.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 2.910937874764204, + "rewards/margins": 5.368858252465725, + "rewards/rejected": -2.4579203579574824, + "step": 20 + }, + { + "entropy": 1.4120451480150222, + "epoch": 0.08888888888888889, + "grad_norm": 0.019287109375, + "learning_rate": 1.4356435643564355e-05, + "logits/chosen": 2.264547569453187, + "logits/rejected": -1.0346384454567032, + "logps/chosen": -64.55358953475952, + "logps/rejected": -259.64974784851074, + "loss": 6.736992509104311e-05, + "mean_token_accuracy": 0.6582182548940182, + "num_tokens": 274816.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.462539532780648, + "rewards/margins": 12.943859684467316, + "rewards/rejected": -8.481320106983185, + "step": 30 + }, + { + "entropy": 1.355741088092327, + "epoch": 0.11851851851851852, + "grad_norm": 0.0004863739013671875, + "learning_rate": 1.930693069306931e-05, + "logits/chosen": 2.3368093516258224, + "logits/rejected": -1.142538018090554, + "logps/chosen": -61.829122233390805, + "logps/rejected": -295.9105268478394, + "loss": 3.1130424758885055e-06, + "mean_token_accuracy": 0.6669795714318753, + "num_tokens": 367268.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.366855585575104, + "rewards/margins": 16.085866248607637, + "rewards/rejected": -11.719010663032531, + "step": 40 + }, + { + "entropy": 1.320019119977951, + "epoch": 0.14814814814814814, + "grad_norm": 0.0001201629638671875, + "learning_rate": 2.4257425742574257e-05, + "logits/chosen": 2.315207261423255, + "logits/rejected": -1.0571934877438707, + "logps/chosen": -63.68361803293228, + "logps/rejected": -302.812979888916, + "loss": 7.578286749776453e-06, + "mean_token_accuracy": 0.6602454461157322, + "num_tokens": 461524.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.11230229139328, + "rewards/margins": 16.546387827396394, + "rewards/rejected": -12.43408569097519, + "step": 50 + }, + { + "entropy": 1.326953475177288, + "epoch": 0.17777777777777778, + "grad_norm": 0.0003871917724609375, + "learning_rate": 2.9207920792079208e-05, + "logits/chosen": 2.2619415245350254, + "logits/rejected": -1.0863394894289136, + "logps/chosen": -63.424767637252806, + "logps/rejected": -304.6159143447876, + "loss": 2.0567724277498202e-06, + "mean_token_accuracy": 0.6664863616228104, + "num_tokens": 555169.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.521066862344742, + "rewards/margins": 17.257024788856505, + "rewards/rejected": -12.735957789421082, + "step": 60 + }, + { + "entropy": 1.3109250903129577, + "epoch": 0.2074074074074074, + "grad_norm": 1.0609626770019531e-05, + "learning_rate": 3.415841584158416e-05, + "logits/chosen": 2.3168210798300413, + "logits/rejected": -1.1463735020520454, + "logps/chosen": -62.93964213132858, + "logps/rejected": -301.49362087249756, + "loss": 2.6633450761437417e-05, + "mean_token_accuracy": 0.6684455543756485, + "num_tokens": 646219.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.369684019684792, + "rewards/margins": 17.029080402851104, + "rewards/rejected": -12.659396553039551, + "step": 70 + }, + { + "entropy": 1.3183406174182892, + "epoch": 0.23703703703703705, + "grad_norm": 8.463859558105469e-06, + "learning_rate": 3.910891089108911e-05, + "logits/chosen": 2.2528845473791295, + "logits/rejected": -1.1451393864081993, + "logps/chosen": -64.70127091407775, + "logps/rejected": -313.31810626983645, + "loss": 2.219452881035977e-07, + "mean_token_accuracy": 0.6682900048792362, + "num_tokens": 741395.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.620437514781952, + "rewards/margins": 18.064030134677886, + "rewards/rejected": -13.443592536449433, + "step": 80 + }, + { + "entropy": 1.306735871732235, + "epoch": 0.26666666666666666, + "grad_norm": 0.00014209747314453125, + "learning_rate": 4.405940594059406e-05, + "logits/chosen": 2.2645358825214656, + "logits/rejected": -1.1447062023763102, + "logps/chosen": -62.47025227546692, + "logps/rejected": -309.65871505737306, + "loss": 6.8541798100341115e-06, + "mean_token_accuracy": 0.6588799960911274, + "num_tokens": 833512.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.536789815127849, + "rewards/margins": 17.808473789691924, + "rewards/rejected": -13.271684014797211, + "step": 90 + }, + { + "entropy": 1.3041012182831764, + "epoch": 0.2962962962962963, + "grad_norm": 0.0001277923583984375, + "learning_rate": 4.9009900990099014e-05, + "logits/chosen": 2.3280876455235577, + "logits/rejected": -1.0593703708195146, + "logps/chosen": -57.693548345565794, + "logps/rejected": -307.1459705352783, + "loss": 2.0000723452540113e-06, + "mean_token_accuracy": 0.6723956800997257, + "num_tokens": 925332.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.472308248281479, + "rewards/margins": 17.747703313827515, + "rewards/rejected": -13.275395059585572, + "step": 100 + }, + { + "epoch": 0.2962962962962963, + "eval_entropy": 1.3091890520149179, + "eval_logits/chosen": 2.2560516885971, + "eval_logits/rejected": -1.1380674279781138, + "eval_logps/chosen": -66.05370368157233, + "eval_logps/rejected": -310.2874222335282, + "eval_loss": 3.520939208101481e-05, + "eval_mean_token_accuracy": 0.662857598358101, + "eval_num_tokens": 925332.0, + "eval_rewards/accuracies": 1.0, + "eval_rewards/chosen": 4.5097555984983915, + "eval_rewards/margins": 18.033509881346376, + "eval_rewards/rejected": -13.523754186563558, + "eval_runtime": 38.3237, + "eval_samples_per_second": 7.437, + "eval_steps_per_second": 3.731, + "step": 100 + }, + { + "entropy": 1.284974904358387, + "epoch": 0.32592592592592595, + "grad_norm": 0.000812530517578125, + "learning_rate": 4.999050767562379e-05, + "logits/chosen": 2.3396127391184693, + "logits/rejected": -1.1138358422485608, + "logps/chosen": -61.88268647193909, + "logps/rejected": -304.68117294311526, + "loss": 3.422158624744043e-06, + "mean_token_accuracy": 0.6786789156496524, + "num_tokens": 1016416.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.462823995947838, + "rewards/margins": 17.767807400226594, + "rewards/rejected": -13.304983520507813, + "step": 110 + }, + { + "entropy": 1.2844461277127266, + "epoch": 0.35555555555555557, + "grad_norm": 0.00010204315185546875, + "learning_rate": 4.9951957461646706e-05, + "logits/chosen": 2.3448500846553584, + "logits/rejected": -1.0939366953296292, + "logps/chosen": -61.62471632957458, + "logps/rejected": -307.099747467041, + "loss": 1.316130146733485e-06, + "mean_token_accuracy": 0.6710287682712078, + "num_tokens": 1105772.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.635418501496315, + "rewards/margins": 18.310668289661407, + "rewards/rejected": -13.675249755382538, + "step": 120 + }, + { + "entropy": 1.2960298083722592, + "epoch": 0.3851851851851852, + "grad_norm": 0.0009765625, + "learning_rate": 4.988380179235843e-05, + "logits/chosen": 2.329744072704608, + "logits/rejected": -1.0829270884851314, + "logps/chosen": -60.90654925107956, + "logps/rejected": -320.0984771728516, + "loss": 2.281175875396002e-06, + "mean_token_accuracy": 0.6639963097870349, + "num_tokens": 1198464.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.473316392302513, + "rewards/margins": 18.291567063331605, + "rewards/rejected": -13.818250668048858, + "step": 130 + }, + { + "entropy": 1.2743454918265342, + "epoch": 0.4148148148148148, + "grad_norm": 0.00115966796875, + "learning_rate": 4.9786121534345265e-05, + "logits/chosen": 2.310538935492521, + "logits/rejected": -1.1208351833476473, + "logps/chosen": -58.558825969696045, + "logps/rejected": -314.1108278274536, + "loss": 1.2155161130067428e-06, + "mean_token_accuracy": 0.674963940680027, + "num_tokens": 1289352.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.255219912528991, + "rewards/margins": 18.211954617500304, + "rewards/rejected": -13.956734585762025, + "step": 140 + }, + { + "entropy": 1.2854739636182786, + "epoch": 0.4444444444444444, + "grad_norm": 0.0001659393310546875, + "learning_rate": 4.965903258506806e-05, + "logits/chosen": 2.33276765322688, + "logits/rejected": -1.1813850704996935, + "logps/chosen": -60.029894495010375, + "logps/rejected": -315.8014057159424, + "loss": 4.05059563490795e-07, + "mean_token_accuracy": 0.6699102215468884, + "num_tokens": 1380079.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.356227645277977, + "rewards/margins": 18.526466822624208, + "rewards/rejected": -14.170239269733429, + "step": 150 + }, + { + "entropy": 1.3013202093541623, + "epoch": 0.4740740740740741, + "grad_norm": 1.0132789611816406e-05, + "learning_rate": 4.950268573535012e-05, + "logits/chosen": 2.3245691177602428, + "logits/rejected": -1.0899626289102018, + "logps/chosen": -62.87379167079926, + "logps/rejected": -325.68464183807373, + "loss": 2.103474116665893e-07, + "mean_token_accuracy": 0.6679917454719544, + "num_tokens": 1474262.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.370475701987743, + "rewards/margins": 18.631532168388368, + "rewards/rejected": -14.261056447029114, + "step": 160 + }, + { + "entropy": 1.2932921536266804, + "epoch": 0.5037037037037037, + "grad_norm": 0.00017070770263671875, + "learning_rate": 4.93172664904641e-05, + "logits/chosen": 2.199208436926322, + "logits/rejected": -1.1313941206494675, + "logps/chosen": -66.81836426258087, + "logps/rejected": -313.0147771835327, + "loss": 5.84564577366109e-07, + "mean_token_accuracy": 0.6591029770672321, + "num_tokens": 1567351.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.2477530300617214, + "rewards/margins": 18.021551287174226, + "rewards/rejected": -13.773798143863678, + "step": 170 + }, + { + "entropy": 1.2906752578914165, + "epoch": 0.5333333333333333, + "grad_norm": 1.1622905731201172e-05, + "learning_rate": 4.9102994850030344e-05, + "logits/chosen": 2.257452431422654, + "logits/rejected": -1.1859710789627558, + "logps/chosen": -60.773719620704654, + "logps/rejected": -319.68780155181884, + "loss": 4.461732260097051e-07, + "mean_token_accuracy": 0.6724127769470215, + "num_tokens": 1659070.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.278084766864777, + "rewards/margins": 18.284436917304994, + "rewards/rejected": -14.006352269649506, + "step": 180 + }, + { + "entropy": 1.3018329843878746, + "epoch": 0.562962962962963, + "grad_norm": 6.4849853515625e-05, + "learning_rate": 4.886012504698769e-05, + "logits/chosen": 2.301720358967305, + "logits/rejected": -1.0489440936301595, + "logps/chosen": -61.81112604141235, + "logps/rejected": -315.3358419418335, + "loss": 5.486400823428994e-07, + "mean_token_accuracy": 0.669282128661871, + "num_tokens": 1750181.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.455310028791428, + "rewards/margins": 18.440570056438446, + "rewards/rejected": -13.985260021686553, + "step": 190 + }, + { + "entropy": 1.3009084612131119, + "epoch": 0.5925925925925926, + "grad_norm": 0.0087890625, + "learning_rate": 4.858894524594653e-05, + "logits/chosen": 2.2154390003090785, + "logits/rejected": -1.1232223850457532, + "logps/chosen": -66.61707370281219, + "logps/rejected": -316.3582260131836, + "loss": 5.881736069568433e-06, + "mean_token_accuracy": 0.6558836951851845, + "num_tokens": 1841453.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.423358170688152, + "rewards/margins": 18.573253047466277, + "rewards/rejected": -14.149895036220551, + "step": 200 + }, + { + "epoch": 0.5925925925925926, + "eval_entropy": 1.294411522108358, + "eval_logits/chosen": 2.25081917507982, + "eval_logits/rejected": -1.1530030385301466, + "eval_logps/chosen": -66.83867399842589, + "eval_logps/rejected": -318.13998082301, + "eval_loss": 1.2471568879846018e-05, + "eval_mean_token_accuracy": 0.6633061449844521, + "eval_num_tokens": 1841453.0, + "eval_rewards/accuracies": 1.0, + "eval_rewards/chosen": 4.431258590904982, + "eval_rewards/margins": 18.740268787304004, + "eval_rewards/rejected": -14.309010145547507, + "eval_runtime": 38.3878, + "eval_samples_per_second": 7.424, + "eval_steps_per_second": 3.725, + "step": 200 + }, + { + "entropy": 1.2944884866476059, + "epoch": 0.6222222222222222, + "grad_norm": 5.125999450683594e-05, + "learning_rate": 4.8289777201281974e-05, + "logits/chosen": 2.3454347980352996, + "logits/rejected": -1.1855823049416294, + "logps/chosen": -64.12132394313812, + "logps/rejected": -325.72925930023194, + "loss": 2.409943363090861e-07, + "mean_token_accuracy": 0.6670842688530684, + "num_tokens": 1935659.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.459967152774334, + "rewards/margins": 19.128938269615173, + "rewards/rejected": -14.668970954418182, + "step": 210 + }, + { + "entropy": 1.2553015358746051, + "epoch": 0.6518518518518519, + "grad_norm": 4.76837158203125e-05, + "learning_rate": 4.796297587537286e-05, + "logits/chosen": 2.308509979398422, + "logits/rejected": -1.0750038709449357, + "logps/chosen": -58.983665919303895, + "logps/rejected": -315.6023241043091, + "loss": 3.415452965782606e-07, + "mean_token_accuracy": 0.6794103197753429, + "num_tokens": 2029268.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.352584506571293, + "rewards/margins": 18.749567639827728, + "rewards/rejected": -14.396983158588409, + "step": 220 + }, + { + "entropy": 1.2749639593064785, + "epoch": 0.6814814814814815, + "grad_norm": 2.7894973754882812e-05, + "learning_rate": 4.760892901743944e-05, + "logits/chosen": 2.263447201357434, + "logits/rejected": -1.0746111454871972, + "logps/chosen": -64.98015263080597, + "logps/rejected": -320.59440212249757, + "loss": 1.3830200259690174e-06, + "mean_token_accuracy": 0.6660085044801235, + "num_tokens": 2123260.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.317273432016373, + "rewards/margins": 18.91294218301773, + "rewards/rejected": -14.59566867351532, + "step": 230 + }, + { + "entropy": 1.2890466079115868, + "epoch": 0.7111111111111111, + "grad_norm": 2.7894973754882812e-05, + "learning_rate": 4.722805670347963e-05, + "logits/chosen": 2.3375335604609835, + "logits/rejected": -1.1522054103145247, + "logps/chosen": -62.949660015106204, + "logps/rejected": -325.21918926239016, + "loss": 1.8574786508906982e-07, + "mean_token_accuracy": 0.6677135609090328, + "num_tokens": 2211888.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.302155223488808, + "rewards/margins": 19.20953975915909, + "rewards/rejected": -14.90738466978073, + "step": 240 + }, + { + "entropy": 1.297672052681446, + "epoch": 0.7407407407407407, + "grad_norm": 1.3649463653564453e-05, + "learning_rate": 4.6820810837849536e-05, + "logits/chosen": 2.3567186399215143, + "logits/rejected": -1.2153430976155177, + "logps/chosen": -61.082769107818606, + "logps/rejected": -327.5022741317749, + "loss": 4.804733634955483e-07, + "mean_token_accuracy": 0.6871423445641994, + "num_tokens": 2307633.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.415084969997406, + "rewards/margins": 19.260318982601166, + "rewards/rejected": -14.845234000682831, + "step": 250 + }, + { + "entropy": 1.289575246721506, + "epoch": 0.7703703703703704, + "grad_norm": 0.00013637542724609375, + "learning_rate": 4.63876746170797e-05, + "logits/chosen": 2.2677862381437905, + "logits/rejected": -1.1125468158062497, + "logps/chosen": -67.2604984998703, + "logps/rejected": -321.5008083343506, + "loss": 1.5667228581150994e-07, + "mean_token_accuracy": 0.6551826104521752, + "num_tokens": 2404339.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.320639500021935, + "rewards/margins": 18.89977525472641, + "rewards/rejected": -14.57913579940796, + "step": 260 + }, + { + "entropy": 1.2907186925411225, + "epoch": 0.8, + "grad_norm": 2.5987625122070312e-05, + "learning_rate": 4.592916195656322e-05, + "logits/chosen": 2.337716368388003, + "logits/rejected": -1.1807755553775976, + "logps/chosen": -61.246209836006166, + "logps/rejected": -321.838525390625, + "loss": 3.219545988031314e-07, + "mean_token_accuracy": 0.67101164534688, + "num_tokens": 2493582.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.55239104628563, + "rewards/margins": 19.164004135131837, + "rewards/rejected": -14.611613166332244, + "step": 270 + }, + { + "entropy": 1.2973388336598872, + "epoch": 0.8296296296296296, + "grad_norm": 1.0728836059570312e-05, + "learning_rate": 4.544581688079602e-05, + "logits/chosen": 2.2622078397191254, + "logits/rejected": -1.1987081179562755, + "logps/chosen": -62.545769691467285, + "logps/rejected": -328.43379402160645, + "loss": 1.6840259604578022e-07, + "mean_token_accuracy": 0.6727346412837505, + "num_tokens": 2583501.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.605609413981438, + "rewards/margins": 19.350289702415466, + "rewards/rejected": -14.744680321216583, + "step": 280 + }, + { + "entropy": 1.270925548672676, + "epoch": 0.8592592592592593, + "grad_norm": 1.4722347259521484e-05, + "learning_rate": 4.4938212877892725e-05, + "logits/chosen": 2.304147609404456, + "logits/rejected": -1.1603299402866345, + "logps/chosen": -64.19616587162018, + "logps/rejected": -321.61998023986814, + "loss": 1.092423531190434e-07, + "mean_token_accuracy": 0.6694903835654259, + "num_tokens": 2673092.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.521212270855903, + "rewards/margins": 19.481856608390807, + "rewards/rejected": -14.960644149780274, + "step": 290 + }, + { + "entropy": 1.2587753705680371, + "epoch": 0.8888888888888888, + "grad_norm": 8.046627044677734e-06, + "learning_rate": 4.4406952219143936e-05, + "logits/chosen": 2.2447284227823454, + "logits/rejected": -1.2047798430182726, + "logps/chosen": -63.220883178710935, + "logps/rejected": -316.7784074783325, + "loss": 9.590096851752606e-07, + "mean_token_accuracy": 0.6734147310256958, + "num_tokens": 2765264.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.453023025393486, + "rewards/margins": 19.15658880472183, + "rewards/rejected": -14.703565871715545, + "step": 300 + }, + { + "epoch": 0.8888888888888888, + "eval_entropy": 1.2864730945833913, + "eval_logits/chosen": 2.245142647914562, + "eval_logits/rejected": -1.1577701959581186, + "eval_logps/chosen": -67.51500383123651, + "eval_logps/rejected": -323.1507350681545, + "eval_loss": 9.1151678134338e-06, + "eval_mean_token_accuracy": 0.6627988360978506, + "eval_num_tokens": 2765264.0, + "eval_rewards/accuracies": 1.0, + "eval_rewards/chosen": 4.3636256072904684, + "eval_rewards/margins": 19.173711116497334, + "eval_rewards/rejected": -14.810085516709547, + "eval_runtime": 38.408, + "eval_samples_per_second": 7.42, + "eval_steps_per_second": 3.723, + "step": 300 + }, + { + "entropy": 1.2768845409154892, + "epoch": 0.9185185185185185, + "grad_norm": 4.315376281738281e-05, + "learning_rate": 4.385266524442241e-05, + "logits/chosen": 2.2955878654382667, + "logits/rejected": -1.200506409819956, + "logps/chosen": -62.47047922611237, + "logps/rejected": -329.1678291320801, + "loss": 2.0756017420353602e-07, + "mean_token_accuracy": 0.6736313477158546, + "num_tokens": 2860290.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.335641312599182, + "rewards/margins": 19.35282862186432, + "rewards/rejected": -15.01718727350235, + "step": 310 + }, + { + "entropy": 1.267455793917179, + "epoch": 0.9481481481481482, + "grad_norm": 2.2649765014648438e-05, + "learning_rate": 4.3276009614285825e-05, + "logits/chosen": 2.301217628396178, + "logits/rejected": -1.1612455940414776, + "logps/chosen": -60.544256925582886, + "logps/rejected": -325.25582218170166, + "loss": 8.637363748675853e-08, + "mean_token_accuracy": 0.6695008479058743, + "num_tokens": 2951326.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.26083664894104, + "rewards/margins": 19.293893361091612, + "rewards/rejected": -15.033056676387787, + "step": 320 + }, + { + "entropy": 1.2802438855171203, + "epoch": 0.9777777777777777, + "grad_norm": 3.1441450119018555e-06, + "learning_rate": 4.267766952966369e-05, + "logits/chosen": 2.3043263280692856, + "logits/rejected": -1.1247859920954724, + "logps/chosen": -65.50963315963745, + "logps/rejected": -327.67705821990967, + "loss": 1.6503474398632533e-06, + "mean_token_accuracy": 0.6599100351333618, + "num_tokens": 3045520.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.30358607918024, + "rewards/margins": 19.168729376792907, + "rewards/rejected": -14.865143322944641, + "step": 330 + }, + { + "entropy": 1.2861412669482983, + "epoch": 1.005925925925926, + "grad_norm": 5.793571472167969e-05, + "learning_rate": 4.205835492005405e-05, + "logits/chosen": 2.319402775016898, + "logits/rejected": -1.2147055357233236, + "logps/chosen": -67.06965022338063, + "logps/rejected": -330.37220262226305, + "loss": 3.5803138871415284e-07, + "mean_token_accuracy": 0.6624507841311003, + "num_tokens": 3130709.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.281106482603048, + "rewards/margins": 19.5897334500363, + "rewards/rejected": -15.308626877634149, + "step": 340 + }, + { + "entropy": 1.2823633886873722, + "epoch": 1.0355555555555556, + "grad_norm": 6.008148193359375e-05, + "learning_rate": 4.141880060119336e-05, + "logits/chosen": 2.339569692570856, + "logits/rejected": -1.147382590046395, + "logps/chosen": -60.31965174674988, + "logps/rejected": -335.1733335494995, + "loss": 1.532446731289383e-07, + "mean_token_accuracy": 0.6735855638980865, + "num_tokens": 3225369.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.492888362705708, + "rewards/margins": 19.827855718135833, + "rewards/rejected": -15.334967303276063, + "step": 350 + }, + { + "entropy": 1.2730201914906503, + "epoch": 1.0651851851851852, + "grad_norm": 0.0001926422119140625, + "learning_rate": 4.075976540319888e-05, + "logits/chosen": 2.3003171439882366, + "logits/rejected": -1.2015335101578284, + "logps/chosen": -60.594508814811704, + "logps/rejected": -329.16415615081786, + "loss": 4.122233804082498e-07, + "mean_token_accuracy": 0.6743721053004265, + "num_tokens": 3318818.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.401582342386246, + "rewards/margins": 19.628432750701904, + "rewards/rejected": -15.22685045003891, + "step": 360 + }, + { + "entropy": 1.2773501306772232, + "epoch": 1.094814814814815, + "grad_norm": 0.000591278076171875, + "learning_rate": 4.0082031270217976e-05, + "logits/chosen": 2.2876778332841776, + "logits/rejected": -1.154664545777339, + "logps/chosen": -63.76930930614471, + "logps/rejected": -319.00229358673096, + "loss": 1.1508095667522866e-06, + "mean_token_accuracy": 0.6718519743531942, + "num_tokens": 3410676.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.197161988914013, + "rewards/margins": 19.0069660782814, + "rewards/rejected": -14.809803891181947, + "step": 370 + }, + { + "entropy": 1.2946350619196891, + "epoch": 1.1244444444444444, + "grad_norm": 0.00022411346435546875, + "learning_rate": 3.9386402332652754e-05, + "logits/chosen": 2.293869972488772, + "logits/rejected": -1.1689247316110847, + "logps/chosen": -64.16428699493409, + "logps/rejected": -339.3799625396729, + "loss": 1.2371945103950566e-07, + "mean_token_accuracy": 0.664111914485693, + "num_tokens": 3504356.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.358540344238281, + "rewards/margins": 19.867380928993224, + "rewards/rejected": -15.508840537071228, + "step": 380 + }, + { + "entropy": 1.2926758706569672, + "epoch": 1.154074074074074, + "grad_norm": 3.6954879760742188e-06, + "learning_rate": 3.867370395306068e-05, + "logits/chosen": 2.1766569425767703, + "logits/rejected": -1.087643152794101, + "logps/chosen": -69.47466747760772, + "logps/rejected": -327.71664810180664, + "loss": 1.194763967760082e-07, + "mean_token_accuracy": 0.6569934625178575, + "num_tokens": 3594396.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.461885306239128, + "rewards/margins": 19.697250258922576, + "rewards/rejected": -15.235364770889282, + "step": 390 + }, + { + "entropy": 1.2546571284532546, + "epoch": 1.1837037037037037, + "grad_norm": 8.940696716308594e-06, + "learning_rate": 3.794478174686328e-05, + "logits/chosen": 2.28344883152218, + "logits/rejected": -1.1421346511748105, + "logps/chosen": -64.66728138923645, + "logps/rejected": -324.4414520263672, + "loss": 7.843665343898465e-08, + "mean_token_accuracy": 0.662892397120595, + "num_tokens": 3682850.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.419300290942192, + "rewards/margins": 19.782128620147706, + "rewards/rejected": -15.362828314304352, + "step": 400 + }, + { + "epoch": 1.1837037037037037, + "eval_entropy": 1.2814960267160322, + "eval_logits/chosen": 2.2466129676027395, + "eval_logits/rejected": -1.158793695801587, + "eval_logps/chosen": -67.91167767398007, + "eval_logps/rejected": -326.0148432804988, + "eval_loss": 7.35781259209034e-06, + "eval_mean_token_accuracy": 0.6629326431067674, + "eval_num_tokens": 3682850.0, + "eval_rewards/accuracies": 1.0, + "eval_rewards/chosen": 4.323958210178189, + "eval_rewards/margins": 19.420454478764032, + "eval_rewards/rejected": -15.096496268585845, + "eval_runtime": 38.5811, + "eval_samples_per_second": 7.387, + "eval_steps_per_second": 3.706, + "step": 400 + }, + { + "entropy": 1.2794287726283073, + "epoch": 1.2133333333333334, + "grad_norm": 6.0498714447021484e-06, + "learning_rate": 3.720050057902496e-05, + "logits/chosen": 2.2348389102512294, + "logits/rejected": -1.20443931083074, + "logps/chosen": -61.86296122074127, + "logps/rejected": -331.40450420379636, + "loss": 1.5817265648365718e-07, + "mean_token_accuracy": 0.6723401784896851, + "num_tokens": 3776120.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.320593364536762, + "rewards/margins": 19.594025874137877, + "rewards/rejected": -15.273432505130767, + "step": 410 + }, + { + "entropy": 1.2816326960921287, + "epoch": 1.242962962962963, + "grad_norm": 2.7865171432495117e-06, + "learning_rate": 3.644174353789204e-05, + "logits/chosen": 2.2707540738720113, + "logits/rejected": -1.191927789160487, + "logps/chosen": -64.48058602809905, + "logps/rejected": -332.1292999267578, + "loss": 9.944698149411124e-08, + "mean_token_accuracy": 0.6633661832660437, + "num_tokens": 3868574.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.300156235694885, + "rewards/margins": 19.600781667232514, + "rewards/rejected": -15.30062551498413, + "step": 420 + }, + { + "entropy": 1.2759614169597626, + "epoch": 1.2725925925925927, + "grad_norm": 6.29425048828125e-05, + "learning_rate": 3.5669410887410095e-05, + "logits/chosen": 2.3054515861422304, + "logits/rejected": -1.1359497002203924, + "logps/chosen": -64.89167664051055, + "logps/rejected": -334.02518730163575, + "loss": 2.404902943453635e-07, + "mean_token_accuracy": 0.6750707887113094, + "num_tokens": 3961842.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.182991920411586, + "rewards/margins": 19.472245895862578, + "rewards/rejected": -15.289253902435302, + "step": 430 + }, + { + "entropy": 1.2799105122685432, + "epoch": 1.3022222222222222, + "grad_norm": 0.00012683868408203125, + "learning_rate": 3.488441899896217e-05, + "logits/chosen": 2.345937521884795, + "logits/rejected": -1.2320921962004756, + "logps/chosen": -58.358681523799895, + "logps/rejected": -320.94243869781496, + "loss": 2.9142654511815635e-07, + "mean_token_accuracy": 0.679117338731885, + "num_tokens": 4051000.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.320470458269119, + "rewards/margins": 19.39275724887848, + "rewards/rejected": -15.072286760807037, + "step": 440 + }, + { + "entropy": 1.2606690004467964, + "epoch": 1.3318518518518518, + "grad_norm": 3.933906555175781e-05, + "learning_rate": 3.4087699264095745e-05, + "logits/chosen": 2.3590029943979434, + "logits/rejected": -1.180752685047298, + "logps/chosen": -57.81958304643631, + "logps/rejected": -327.3783897399902, + "loss": 1.31027945826645e-07, + "mean_token_accuracy": 0.6829047620296478, + "num_tokens": 4144901.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.355239224433899, + "rewards/margins": 19.49090631008148, + "rewards/rejected": -15.135667026042938, + "step": 450 + }, + { + "entropy": 1.2720958605408668, + "epoch": 1.3614814814814815, + "grad_norm": 7.152557373046875e-06, + "learning_rate": 3.328019698942826e-05, + "logits/chosen": 2.28385155999331, + "logits/rejected": -1.0655708622856899, + "logps/chosen": -65.57811061143875, + "logps/rejected": -324.391535949707, + "loss": 1.4767925904379808e-07, + "mean_token_accuracy": 0.6687033347785473, + "num_tokens": 4235782.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.3411215707659725, + "rewards/margins": 19.290599715709686, + "rewards/rejected": -14.949478232860566, + "step": 460 + }, + { + "entropy": 1.2874604403972625, + "epoch": 1.3911111111111112, + "grad_norm": 0.00011920928955078125, + "learning_rate": 3.246287027504237e-05, + "logits/chosen": 2.2650682684340646, + "logits/rejected": -1.1018559007875284, + "logps/chosen": -63.989255738258365, + "logps/rejected": -333.0100872039795, + "loss": 2.971911953864037e-07, + "mean_token_accuracy": 0.6702111043035984, + "num_tokens": 4327602.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.187621220946312, + "rewards/margins": 19.38519823551178, + "rewards/rejected": -15.197577047348023, + "step": 470 + }, + { + "entropy": 1.293594116717577, + "epoch": 1.4207407407407406, + "grad_norm": 9.870529174804688e-05, + "learning_rate": 3.163668887770181e-05, + "logits/chosen": 2.306861956625707, + "logits/rejected": -1.1661940995045186, + "logps/chosen": -61.81727409362793, + "logps/rejected": -332.9074993133545, + "loss": 2.5863685095828257e-07, + "mean_token_accuracy": 0.6736553743481636, + "num_tokens": 4419888.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.17706933170557, + "rewards/margins": 19.331134915351868, + "rewards/rejected": -15.154065454006195, + "step": 480 + }, + { + "entropy": 1.2759695634245873, + "epoch": 1.4503703703703703, + "grad_norm": 6.771087646484375e-05, + "learning_rate": 3.08026330602367e-05, + "logits/chosen": 2.338572971372103, + "logits/rejected": -1.1794468686720447, + "logps/chosen": -58.53629996776581, + "logps/rejected": -332.7186403274536, + "loss": 8.574451726417465e-08, + "mean_token_accuracy": 0.6770360998809337, + "num_tokens": 4511414.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.380688160657883, + "rewards/margins": 19.57960090637207, + "rewards/rejected": -15.198912739753723, + "step": 490 + }, + { + "entropy": 1.2591664418578148, + "epoch": 1.48, + "grad_norm": 8.046627044677734e-06, + "learning_rate": 2.9961692428463285e-05, + "logits/chosen": 2.2509338882701817, + "logits/rejected": -1.136340958920119, + "logps/chosen": -66.00177557468415, + "logps/rejected": -332.10723934173586, + "loss": 3.2082580219139346e-07, + "mean_token_accuracy": 0.6653452910482883, + "num_tokens": 4604818.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.144159619510174, + "rewards/margins": 19.529365801811217, + "rewards/rejected": -15.385206091403962, + "step": 500 + }, + { + "epoch": 1.48, + "eval_entropy": 1.2810596311009013, + "eval_logits/chosen": 2.2449733761384874, + "eval_logits/rejected": -1.1597314363523379, + "eval_logps/chosen": -67.92375413854639, + "eval_logps/rejected": -326.1967938829969, + "eval_loss": 7.327445018745493e-06, + "eval_mean_token_accuracy": 0.6633227342492217, + "eval_num_tokens": 4604818.0, + "eval_rewards/accuracies": 1.0, + "eval_rewards/chosen": 4.322750580894364, + "eval_rewards/margins": 19.43744218599546, + "eval_rewards/rejected": -15.114691607601992, + "eval_runtime": 38.6636, + "eval_samples_per_second": 7.371, + "eval_steps_per_second": 3.699, + "step": 500 + }, + { + "entropy": 1.2616984777152538, + "epoch": 1.5096296296296297, + "grad_norm": 0.000125885009765625, + "learning_rate": 2.9114864757018352e-05, + "logits/chosen": 2.294192165543051, + "logits/rejected": -1.0944581394998258, + "logps/chosen": -64.26435973644257, + "logps/rejected": -325.8657173156738, + "loss": 1.0070654070659657e-07, + "mean_token_accuracy": 0.6660138607025147, + "num_tokens": 4695387.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.35933136343956, + "rewards/margins": 19.695837664604188, + "rewards/rejected": -15.336506283283233, + "step": 510 + }, + { + "entropy": 1.2706036016345024, + "epoch": 1.5392592592592593, + "grad_norm": 8.821487426757812e-05, + "learning_rate": 2.8263154805501297e-05, + "logits/chosen": 2.228209829000261, + "logits/rejected": -1.113362437458028, + "logps/chosen": -69.59165489673615, + "logps/rejected": -321.3457368850708, + "loss": 1.4221999435903854e-07, + "mean_token_accuracy": 0.6667736105620861, + "num_tokens": 4788711.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.314350298047065, + "rewards/margins": 19.27485271692276, + "rewards/rejected": -14.960502445697784, + "step": 520 + }, + { + "entropy": 1.2767572030425072, + "epoch": 1.568888888888889, + "grad_norm": 0.0002460479736328125, + "learning_rate": 2.7407573126328545e-05, + "logits/chosen": 2.3174187011624654, + "logits/rejected": -1.2211792118899774, + "logps/chosen": -58.480189538002016, + "logps/rejected": -321.1500442504883, + "loss": 1.383585185976699e-07, + "mean_token_accuracy": 0.675158528238535, + "num_tokens": 4877592.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.162171944975853, + "rewards/margins": 19.111147022247316, + "rewards/rejected": -14.948975014686585, + "step": 530 + }, + { + "entropy": 1.2621485769748688, + "epoch": 1.5985185185185187, + "grad_norm": 3.844499588012695e-06, + "learning_rate": 2.6549134865714868e-05, + "logits/chosen": 2.26315295877127, + "logits/rejected": -1.12053805020359, + "logps/chosen": -64.27606203556061, + "logps/rejected": -328.73465633392334, + "loss": 1.7509564713691362e-07, + "mean_token_accuracy": 0.670067022740841, + "num_tokens": 4969700.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.281642350554466, + "rewards/margins": 19.606130039691926, + "rewards/rejected": -15.32448751926422, + "step": 540 + }, + { + "entropy": 1.266157256066799, + "epoch": 1.6281481481481481, + "grad_norm": 1.4901161193847656e-05, + "learning_rate": 2.5688858559204053e-05, + "logits/chosen": 2.3036601533908074, + "logits/rejected": -1.1204950201201778, + "logps/chosen": -63.52790377140045, + "logps/rejected": -332.37768840789795, + "loss": 1.9515678104653488e-07, + "mean_token_accuracy": 0.6726867496967316, + "num_tokens": 5064766.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.242260693013668, + "rewards/margins": 19.574455058574678, + "rewards/rejected": -15.33219426870346, + "step": 550 + }, + { + "entropy": 1.272486585378647, + "epoch": 1.6577777777777778, + "grad_norm": 3.933906555175781e-06, + "learning_rate": 2.4827764923178246e-05, + "logits/chosen": 2.3149282625691305, + "logits/rejected": -1.2307484890552138, + "logps/chosen": -60.608607578277585, + "logps/rejected": -325.5159580230713, + "loss": 1.471338123337773e-07, + "mean_token_accuracy": 0.6735777668654919, + "num_tokens": 5155008.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.4290194630622866, + "rewards/margins": 19.601059877872466, + "rewards/rejected": -15.172040581703186, + "step": 560 + }, + { + "entropy": 1.2813369497656821, + "epoch": 1.6874074074074072, + "grad_norm": 0.000339508056640625, + "learning_rate": 2.3966875643779667e-05, + "logits/chosen": 2.2549751619675136, + "logits/rejected": -1.1594325055530381, + "logps/chosen": -64.34185855388641, + "logps/rejected": -327.56716537475586, + "loss": 1.8051871393254259e-07, + "mean_token_accuracy": 0.6600819997489452, + "num_tokens": 5248360.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.4127004653215405, + "rewards/margins": 19.473870050907134, + "rewards/rejected": -15.061169505119324, + "step": 570 + }, + { + "entropy": 1.2690633878111839, + "epoch": 1.717037037037037, + "grad_norm": 0.000118255615234375, + "learning_rate": 2.3107212164681776e-05, + "logits/chosen": 2.277724768747875, + "logits/rejected": -1.1787549020085861, + "logps/chosen": -66.66345942020416, + "logps/rejected": -323.8808866500855, + "loss": 2.298829940627911e-07, + "mean_token_accuracy": 0.6669816076755524, + "num_tokens": 5340554.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.2985071912407875, + "rewards/margins": 19.356661868095397, + "rewards/rejected": -15.058154690265656, + "step": 580 + }, + { + "entropy": 1.303315782546997, + "epoch": 1.7466666666666666, + "grad_norm": 9.894371032714844e-06, + "learning_rate": 2.224979447514802e-05, + "logits/chosen": 2.2417929976862743, + "logits/rejected": -1.1242295347690896, + "logps/chosen": -66.85722150802613, + "logps/rejected": -326.75849361419677, + "loss": 5.121967660670635e-07, + "mean_token_accuracy": 0.6579514071345329, + "num_tokens": 5433019.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.279269021749497, + "rewards/margins": 19.181958985328674, + "rewards/rejected": -14.902689921855927, + "step": 590 + }, + { + "entropy": 1.2511712044477463, + "epoch": 1.7762962962962963, + "grad_norm": 1.811981201171875e-05, + "learning_rate": 2.1395639899816333e-05, + "logits/chosen": 2.4288169740201644, + "logits/rejected": -1.0983362881453427, + "logps/chosen": -58.963488447666165, + "logps/rejected": -328.273935508728, + "loss": 5.187651368032676e-07, + "mean_token_accuracy": 0.6844911381602288, + "num_tokens": 5527564.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.607324048876762, + "rewards/margins": 19.77645412683487, + "rewards/rejected": -15.169130110740662, + "step": 600 + }, + { + "epoch": 1.7762962962962963, + "eval_entropy": 1.2805373877078503, + "eval_logits/chosen": 2.243851235815972, + "eval_logits/rejected": -1.1594670058021077, + "eval_logps/chosen": -67.99770445923706, + "eval_logps/rejected": -326.55811144421983, + "eval_loss": 6.923921773704933e-06, + "eval_mean_token_accuracy": 0.6634519100189209, + "eval_num_tokens": 5527564.0, + "eval_rewards/accuracies": 1.0, + "eval_rewards/chosen": 4.315355520148377, + "eval_rewards/margins": 19.466178794007202, + "eval_rewards/rejected": -15.15082325968709, + "eval_runtime": 39.0736, + "eval_samples_per_second": 7.294, + "eval_steps_per_second": 3.66, + "step": 600 + }, + { + "entropy": 1.2721578769385815, + "epoch": 1.805925925925926, + "grad_norm": 1.3053417205810547e-05, + "learning_rate": 2.054576189164518e-05, + "logits/chosen": 2.3202715445664515, + "logits/rejected": -1.132963813611814, + "logps/chosen": -60.805744123458865, + "logps/rejected": -332.585825920105, + "loss": 3.902006255884771e-08, + "mean_token_accuracy": 0.6726082235574722, + "num_tokens": 5620386.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.585019749403, + "rewards/margins": 20.017272901535033, + "rewards/rejected": -15.43225302696228, + "step": 610 + }, + { + "entropy": 1.2652241297066211, + "epoch": 1.8355555555555556, + "grad_norm": 1.633167266845703e-05, + "learning_rate": 1.9701168829453306e-05, + "logits/chosen": 2.259219360616968, + "logits/rejected": -1.0503303018419232, + "logps/chosen": -66.24489846229554, + "logps/rejected": -322.5876424789429, + "loss": 1.7130523701780476e-07, + "mean_token_accuracy": 0.6619290798902512, + "num_tokens": 5712637.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.140004143118858, + "rewards/margins": 19.201012241840363, + "rewards/rejected": -15.06100798845291, + "step": 620 + }, + { + "entropy": 1.2682488977909088, + "epoch": 1.8651851851851853, + "grad_norm": 2.956390380859375e-05, + "learning_rate": 1.8862862821480025e-05, + "logits/chosen": 2.3071657025101677, + "logits/rejected": -1.1965660990092328, + "logps/chosen": -65.7512573480606, + "logps/rejected": -326.63800010681155, + "loss": 9.80972572506289e-08, + "mean_token_accuracy": 0.6676984764635563, + "num_tokens": 5804212.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.233162444829941, + "rewards/margins": 19.35871180295944, + "rewards/rejected": -15.125549244880677, + "step": 630 + }, + { + "entropy": 1.2738682702183723, + "epoch": 1.894814814814815, + "grad_norm": 1.3709068298339844e-06, + "learning_rate": 1.8031838516385423e-05, + "logits/chosen": 2.277379836690332, + "logits/rejected": -1.1057443809996192, + "logps/chosen": -64.37387623786927, + "logps/rejected": -329.1884712219238, + "loss": 5.206376272326452e-08, + "mean_token_accuracy": 0.6642427556216717, + "num_tokens": 5897138.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.30352343916893, + "rewards/margins": 19.877286291122438, + "rewards/rejected": -15.573762822151185, + "step": 640 + }, + { + "entropy": 1.2760708816349506, + "epoch": 1.9244444444444444, + "grad_norm": 5.1021575927734375e-05, + "learning_rate": 1.7209081923101472e-05, + "logits/chosen": 2.327136815796652, + "logits/rejected": -1.1425043440165608, + "logps/chosen": -66.02759101390839, + "logps/rejected": -332.0861042022705, + "loss": 1.4513241239910712e-07, + "mean_token_accuracy": 0.66774348244071, + "num_tokens": 5990180.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.311979895830154, + "rewards/margins": 19.720207822322845, + "rewards/rejected": -15.408227932453155, + "step": 650 + }, + { + "entropy": 1.2696824759244918, + "epoch": 1.954074074074074, + "grad_norm": 2.4557113647460938e-05, + "learning_rate": 1.639556924093404e-05, + "logits/chosen": 2.2885721283972242, + "logits/rejected": -1.12765834632216, + "logps/chosen": -64.96237716674804, + "logps/rejected": -326.4824584960937, + "loss": 1.2898549357487353e-07, + "mean_token_accuracy": 0.6572523653507233, + "num_tokens": 6084186.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.2255510374903675, + "rewards/margins": 19.46041429042816, + "rewards/rejected": -15.234863257408142, + "step": 660 + }, + { + "entropy": 1.2641157731413841, + "epoch": 1.9837037037037037, + "grad_norm": 1.9073486328125e-06, + "learning_rate": 1.5592265701304114e-05, + "logits/chosen": 2.3036376802600183, + "logits/rejected": -1.1734117772095596, + "logps/chosen": -65.30954070091248, + "logps/rejected": -323.9220626831055, + "loss": 2.2752515178581233e-07, + "mean_token_accuracy": 0.6635996960103512, + "num_tokens": 6177151.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.288326196372509, + "rewards/margins": 19.406356739997864, + "rewards/rejected": -15.118030607700348, + "step": 670 + }, + { + "entropy": 1.2779945597836846, + "epoch": 2.011851851851852, + "grad_norm": 0.000255584716796875, + "learning_rate": 1.4800124422502334e-05, + "logits/chosen": 2.3972204176263543, + "logits/rejected": -1.1209583839266675, + "logps/chosen": -61.41229806448284, + "logps/rejected": -324.14299693860505, + "loss": 3.6133037610852626e-07, + "mean_token_accuracy": 0.6630792186448449, + "num_tokens": 6262744.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.139316172976243, + "rewards/margins": 19.215763267717865, + "rewards/rejected": -15.076447160620438, + "step": 680 + }, + { + "entropy": 1.266596408188343, + "epoch": 2.0414814814814815, + "grad_norm": 4.4889748096466064e-07, + "learning_rate": 1.4020085278815745e-05, + "logits/chosen": 2.3154754788175156, + "logits/rejected": -1.152066471886049, + "logps/chosen": -62.28092246055603, + "logps/rejected": -331.86610050201415, + "loss": 7.074050927258213e-08, + "mean_token_accuracy": 0.6724916785955429, + "num_tokens": 6357968.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.411373123526573, + "rewards/margins": 19.874622988700867, + "rewards/rejected": -15.4632497549057, + "step": 690 + }, + { + "entropy": 1.2671823538839817, + "epoch": 2.071111111111111, + "grad_norm": 0.0002994537353515625, + "learning_rate": 1.3253073785368547e-05, + "logits/chosen": 2.29187693793979, + "logits/rejected": -1.1263780908242413, + "logps/chosen": -65.86139830350876, + "logps/rejected": -331.38874130249025, + "loss": 2.163802719223895e-07, + "mean_token_accuracy": 0.6635798320174218, + "num_tokens": 6449816.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.283302050828934, + "rewards/margins": 19.748378670215608, + "rewards/rejected": -15.465076541900634, + "step": 700 + }, + { + "epoch": 2.071111111111111, + "eval_entropy": 1.2803548124286679, + "eval_logits/chosen": 2.243403260803776, + "eval_logits/rejected": -1.1587966303436696, + "eval_logps/chosen": -68.02522693980823, + "eval_logps/rejected": -326.6569073016827, + "eval_loss": 7.350444775511278e-06, + "eval_mean_token_accuracy": 0.6639135730016482, + "eval_num_tokens": 6449816.0, + "eval_rewards/accuracies": 1.0, + "eval_rewards/chosen": 4.312603289847607, + "eval_rewards/margins": 19.473306129028746, + "eval_rewards/rejected": -15.160702772073812, + "eval_runtime": 39.1309, + "eval_samples_per_second": 7.283, + "eval_steps_per_second": 3.654, + "step": 700 + }, + { + "entropy": 1.2728232815861702, + "epoch": 2.100740740740741, + "grad_norm": 8.821487426757812e-05, + "learning_rate": 1.2500000000000006e-05, + "logits/chosen": 2.255679331290835, + "logits/rejected": -1.1216202176026864, + "logps/chosen": -61.825205373764035, + "logps/rejected": -323.78655910491943, + "loss": 2.141799996024929e-07, + "mean_token_accuracy": 0.667589108645916, + "num_tokens": 6540123.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.437801697850228, + "rewards/margins": 19.34732162952423, + "rewards/rejected": -14.90951988697052, + "step": 710 + }, + { + "entropy": 1.271144437789917, + "epoch": 2.1303703703703705, + "grad_norm": 6.771087646484375e-05, + "learning_rate": 1.1761757443482286e-05, + "logits/chosen": 2.2690192203566064, + "logits/rejected": -1.176060724310163, + "logps/chosen": -65.25097322463989, + "logps/rejected": -325.64191341400146, + "loss": 7.844523679523263e-08, + "mean_token_accuracy": 0.6650686420500278, + "num_tokens": 6629998.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.375448957085609, + "rewards/margins": 19.631182181835175, + "rewards/rejected": -15.255733144283294, + "step": 720 + }, + { + "entropy": 1.263370680809021, + "epoch": 2.16, + "grad_norm": 0.00067901611328125, + "learning_rate": 1.1039222039359645e-05, + "logits/chosen": 2.2999157883359946, + "logits/rejected": -1.1241040776625622, + "logps/chosen": -65.3091173171997, + "logps/rejected": -327.6589548110962, + "loss": 6.326518359855982e-07, + "mean_token_accuracy": 0.6666021130979061, + "num_tokens": 6721579.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.218965843319893, + "rewards/margins": 19.43423910140991, + "rewards/rejected": -15.21527328491211, + "step": 730 + }, + { + "entropy": 1.2835220411419868, + "epoch": 2.18962962962963, + "grad_norm": 9.059906005859375e-06, + "learning_rate": 1.0333251074666609e-05, + "logits/chosen": 2.2917566691940756, + "logits/rejected": -1.2816091428578473, + "logps/chosen": -66.77954509258271, + "logps/rejected": -333.47667198181153, + "loss": 1.7912802832142916e-07, + "mean_token_accuracy": 0.6699437476694584, + "num_tokens": 6813377.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.310945153236389, + "rewards/margins": 19.620563447475433, + "rewards/rejected": -15.309618008136749, + "step": 740 + }, + { + "entropy": 1.2834552891552449, + "epoch": 2.2192592592592595, + "grad_norm": 2.8252601623535156e-05, + "learning_rate": 9.644682182758306e-06, + "logits/chosen": 2.296253967390375, + "logits/rejected": -1.1382098734020025, + "logps/chosen": -65.18716852664947, + "logps/rejected": -333.20018062591555, + "loss": 3.619171593527426e-07, + "mean_token_accuracy": 0.670500548928976, + "num_tokens": 6906265.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.34758489727974, + "rewards/margins": 19.480838096141817, + "rewards/rejected": -15.133253240585328, + "step": 750 + }, + { + "entropy": 1.2889503717422486, + "epoch": 2.2488888888888887, + "grad_norm": 3.6716461181640625e-05, + "learning_rate": 8.974332349459994e-06, + "logits/chosen": 2.274084705882701, + "logits/rejected": -1.1973438289074938, + "logps/chosen": -62.72564001083374, + "logps/rejected": -335.81445999145507, + "loss": 2.772670995909721e-07, + "mean_token_accuracy": 0.663724122941494, + "num_tokens": 7004404.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.22343917787075, + "rewards/margins": 19.607576775550843, + "rewards/rejected": -15.384137523174285, + "step": 760 + }, + { + "entropy": 1.2709729745984077, + "epoch": 2.2785185185185184, + "grad_norm": 6.818771362304688e-05, + "learning_rate": 8.322996943714672e-06, + "logits/chosen": 2.315020811648067, + "logits/rejected": -1.1844662490433007, + "logps/chosen": -60.06333501338959, + "logps/rejected": -324.07994575500487, + "loss": 6.899633149259899e-08, + "mean_token_accuracy": 0.674921452999115, + "num_tokens": 7091630.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.219157862663269, + "rewards/margins": 19.409486842155456, + "rewards/rejected": -15.190328931808471, + "step": 770 + }, + { + "entropy": 1.2802273795008658, + "epoch": 2.308148148148148, + "grad_norm": 7.677078247070312e-05, + "learning_rate": 7.691448773879257e-06, + "logits/chosen": 2.25499691013824, + "logits/rejected": -1.1512603749381776, + "logps/chosen": -63.52460696697235, + "logps/rejected": -325.2589776992798, + "loss": 1.4859768953101594e-07, + "mean_token_accuracy": 0.6620502531528473, + "num_tokens": 7182707.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.2716171652078625, + "rewards/margins": 19.37758446931839, + "rewards/rejected": -15.10596705675125, + "step": 780 + }, + { + "entropy": 1.2827171966433526, + "epoch": 2.3377777777777777, + "grad_norm": 6.645917892456055e-06, + "learning_rate": 7.080437170788723e-06, + "logits/chosen": 2.311279925102533, + "logits/rejected": -1.1746961520146812, + "logps/chosen": -59.46050326824188, + "logps/rejected": -336.5900556564331, + "loss": 3.455784508332727e-07, + "mean_token_accuracy": 0.6743544064462185, + "num_tokens": 7275656.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.199921746551991, + "rewards/margins": 19.863027906417848, + "rewards/rejected": -15.663106298446655, + "step": 790 + }, + { + "entropy": 1.2675680316984654, + "epoch": 2.3674074074074074, + "grad_norm": 0.00015926361083984375, + "learning_rate": 6.490687098676332e-06, + "logits/chosen": 2.3366430778544096, + "logits/rejected": -1.1229084532499813, + "logps/chosen": -58.34785094261169, + "logps/rejected": -331.7050970077515, + "loss": 6.806028522987617e-07, + "mean_token_accuracy": 0.6765504986047745, + "num_tokens": 7367211.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.387070021033287, + "rewards/margins": 19.79595556259155, + "rewards/rejected": -15.40888546705246, + "step": 800 + }, + { + "epoch": 2.3674074074074074, + "eval_entropy": 1.2803315270197142, + "eval_logits/chosen": 2.2436578263578553, + "eval_logits/rejected": -1.158513689404391, + "eval_logps/chosen": -68.02018330980847, + "eval_logps/rejected": -326.71273963768164, + "eval_loss": 7.024308615655173e-06, + "eval_mean_token_accuracy": 0.6631912376497175, + "eval_num_tokens": 7367211.0, + "eval_rewards/accuracies": 1.0, + "eval_rewards/chosen": 4.313107631006441, + "eval_rewards/margins": 19.479393625592852, + "eval_rewards/rejected": -15.166285921643665, + "eval_runtime": 39.4139, + "eval_samples_per_second": 7.231, + "eval_steps_per_second": 3.628, + "step": 800 + }, + { + "entropy": 1.2536483012139796, + "epoch": 2.397037037037037, + "grad_norm": 0.00010061264038085938, + "learning_rate": 5.9228982950048416e-06, + "logits/chosen": 2.306488303700283, + "logits/rejected": -1.102614074804673, + "logps/chosen": -64.7256872177124, + "logps/rejected": -316.81419982910154, + "loss": 2.7541559575183785e-07, + "mean_token_accuracy": 0.6695533588528633, + "num_tokens": 7456110.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.3090651541948315, + "rewards/margins": 19.122487926483153, + "rewards/rejected": -14.813422644138337, + "step": 810 + }, + { + "entropy": 1.2691614858806133, + "epoch": 2.4266666666666667, + "grad_norm": 3.844499588012695e-06, + "learning_rate": 5.377744440229135e-06, + "logits/chosen": 2.276143363869749, + "logits/rejected": -1.2205610514710212, + "logps/chosen": -64.77891628742218, + "logps/rejected": -330.2338497161865, + "loss": 9.86900658972445e-08, + "mean_token_accuracy": 0.6663463093340397, + "num_tokens": 7548949.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.394037923216819, + "rewards/margins": 19.882170844078065, + "rewards/rejected": -15.488132905960082, + "step": 820 + }, + { + "entropy": 1.2759164057672023, + "epoch": 2.4562962962962964, + "grad_norm": 1.5735626220703125e-05, + "learning_rate": 4.855872358475547e-06, + "logits/chosen": 2.2611198054529975, + "logits/rejected": -1.1668456534925964, + "logps/chosen": -63.698548150062564, + "logps/rejected": -323.2269037246704, + "loss": 1.5968792013154598e-07, + "mean_token_accuracy": 0.6731547065079212, + "num_tokens": 7637248.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.312624426186085, + "rewards/margins": 19.461497557163238, + "rewards/rejected": -15.148873174190522, + "step": 830 + }, + { + "entropy": 1.2611838012933732, + "epoch": 2.485925925925926, + "grad_norm": 0.0002841949462890625, + "learning_rate": 4.357901250086108e-06, + "logits/chosen": 2.2900714239429774, + "logits/rejected": -1.2470973023927523, + "logps/chosen": -62.78718073368073, + "logps/rejected": -324.9568237304687, + "loss": 1.4255396081352955e-07, + "mean_token_accuracy": 0.6746141783893108, + "num_tokens": 7726306.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.291703987121582, + "rewards/margins": 19.421786904335022, + "rewards/rejected": -15.130082988739014, + "step": 840 + }, + { + "entropy": 1.2697825357317924, + "epoch": 2.5155555555555553, + "grad_norm": 0.0002613067626953125, + "learning_rate": 3.884421956938378e-06, + "logits/chosen": 2.3862500773086817, + "logits/rejected": -1.1250058733609287, + "logps/chosen": -61.92496125698089, + "logps/rejected": -327.44327507019045, + "loss": 1.493016384301882e-07, + "mean_token_accuracy": 0.6789508298039436, + "num_tokens": 7822125.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.364183455705643, + "rewards/margins": 19.506937170028685, + "rewards/rejected": -15.142753863334656, + "step": 850 + }, + { + "entropy": 1.2504698053002357, + "epoch": 2.5451851851851854, + "grad_norm": 4.506111145019531e-05, + "learning_rate": 3.435996261412591e-06, + "logits/chosen": 2.2986260733744173, + "logits/rejected": -1.1058551351474077, + "logps/chosen": -65.89442076683045, + "logps/rejected": -330.00583362579346, + "loss": 1.4223222706277737e-07, + "mean_token_accuracy": 0.6768516503274441, + "num_tokens": 7915620.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.385465979576111, + "rewards/margins": 20.04776966571808, + "rewards/rejected": -15.662303495407105, + "step": 860 + }, + { + "entropy": 1.2795700900256635, + "epoch": 2.5748148148148147, + "grad_norm": 2.5153160095214844e-05, + "learning_rate": 3.013156219837776e-06, + "logits/chosen": 2.237134730489094, + "logits/rejected": -1.1218436377455188, + "logps/chosen": -65.73119521141052, + "logps/rejected": -322.3418972015381, + "loss": 2.536130750740995e-07, + "mean_token_accuracy": 0.665914348512888, + "num_tokens": 8006742.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.355473381280899, + "rewards/margins": 19.215320122241973, + "rewards/rejected": -14.85984662771225, + "step": 870 + }, + { + "entropy": 1.2761830180883407, + "epoch": 2.6044444444444443, + "grad_norm": 1.1742115020751953e-05, + "learning_rate": 2.6164035312078448e-06, + "logits/chosen": 2.2800606568463015, + "logits/rejected": -1.1315629358769943, + "logps/chosen": -61.99486291408539, + "logps/rejected": -326.9346130371094, + "loss": 2.3778172817401355e-07, + "mean_token_accuracy": 0.6662385709583759, + "num_tokens": 8100018.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.3306570738554, + "rewards/margins": 19.479354202747345, + "rewards/rejected": -15.148697006702424, + "step": 880 + }, + { + "entropy": 1.279340496659279, + "epoch": 2.634074074074074, + "grad_norm": 9.632110595703125e-05, + "learning_rate": 2.2462089419165777e-06, + "logits/chosen": 2.3029893365635097, + "logits/rejected": -1.171205228496424, + "logps/chosen": -65.00987663269044, + "logps/rejected": -330.74864501953124, + "loss": 6.256735787246726e-08, + "mean_token_accuracy": 0.6665534779429436, + "num_tokens": 8194303.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.474057069420814, + "rewards/margins": 19.85085200071335, + "rewards/rejected": -15.376794970035553, + "step": 890 + }, + { + "entropy": 1.2603400394320488, + "epoch": 2.6637037037037037, + "grad_norm": 2.4557113647460938e-05, + "learning_rate": 1.9030116872178316e-06, + "logits/chosen": 2.249223475648134, + "logits/rejected": -1.0936998011992798, + "logps/chosen": -64.42266466617585, + "logps/rejected": -319.57002086639403, + "loss": 1.4027624501977697e-07, + "mean_token_accuracy": 0.6666488841176033, + "num_tokens": 8285684.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.301889230310917, + "rewards/margins": 19.297891116142274, + "rewards/rejected": -14.99600180387497, + "step": 900 + }, + { + "epoch": 2.6637037037037037, + "eval_entropy": 1.2802694969243937, + "eval_logits/chosen": 2.2445470870400355, + "eval_logits/rejected": -1.159552092885801, + "eval_logps/chosen": -67.97919230694538, + "eval_logps/rejected": -326.7502327232094, + "eval_loss": 6.49735875413171e-06, + "eval_mean_token_accuracy": 0.6639414579718264, + "eval_num_tokens": 8285684.0, + "eval_rewards/accuracies": 1.0, + "eval_rewards/chosen": 4.317206760803303, + "eval_rewards/margins": 19.48724203843337, + "eval_rewards/rejected": -15.17003523553168, + "eval_runtime": 39.451, + "eval_samples_per_second": 7.224, + "eval_steps_per_second": 3.625, + "step": 900 + }, + { + "entropy": 1.2844877071678638, + "epoch": 2.6933333333333334, + "grad_norm": 2.205371856689453e-05, + "learning_rate": 1.5872189700736339e-06, + "logits/chosen": 2.341007675682582, + "logits/rejected": -1.1647508343869704, + "logps/chosen": -65.77681732177734, + "logps/rejected": -333.8214714050293, + "loss": 9.888794920698274e-08, + "mean_token_accuracy": 0.6609948001801967, + "num_tokens": 8379162.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.243448624014855, + "rewards/margins": 19.678018867969513, + "rewards/rejected": -15.434570217132569, + "step": 910 + }, + { + "entropy": 1.2820228539407252, + "epoch": 2.722962962962963, + "grad_norm": 2.9325485229492188e-05, + "learning_rate": 1.2992054780085694e-06, + "logits/chosen": 2.308397402508807, + "logits/rejected": -1.1578754052596163, + "logps/chosen": -60.28462275266647, + "logps/rejected": -329.8702238082886, + "loss": 5.674120302501251e-08, + "mean_token_accuracy": 0.6794499330222606, + "num_tokens": 8471171.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.262164607644081, + "rewards/margins": 19.57164797782898, + "rewards/rejected": -15.309483373165131, + "step": 920 + }, + { + "entropy": 1.2623218022286893, + "epoch": 2.7525925925925927, + "grad_norm": 0.00011157989501953125, + "learning_rate": 1.0393129385436824e-06, + "logits/chosen": 2.3240682024976924, + "logits/rejected": -1.0668921112986016, + "logps/chosen": -59.67337925434113, + "logps/rejected": -327.08252410888673, + "loss": 1.114461042561743e-07, + "mean_token_accuracy": 0.6755911834537983, + "num_tokens": 8562918.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.329451468586922, + "rewards/margins": 19.44830514192581, + "rewards/rejected": -15.118853759765624, + "step": 930 + }, + { + "entropy": 1.2947206407785417, + "epoch": 2.7822222222222224, + "grad_norm": 0.00011396408081054688, + "learning_rate": 8.078497137373242e-07, + "logits/chosen": 2.221305657513471, + "logits/rejected": -1.151073296194637, + "logps/chosen": -66.11703178882598, + "logps/rejected": -329.9900022506714, + "loss": 1.417975795448001e-07, + "mean_token_accuracy": 0.6663264043629169, + "num_tokens": 8655421.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.399758037924767, + "rewards/margins": 19.651811373233794, + "rewards/rejected": -15.252053284645081, + "step": 940 + }, + { + "entropy": 1.2984899178147316, + "epoch": 2.811851851851852, + "grad_norm": 0.00030517578125, + "learning_rate": 6.050904343141095e-07, + "logits/chosen": 2.22722939090554, + "logits/rejected": -1.1332568821088258, + "logps/chosen": -66.14050345420837, + "logps/rejected": -324.22646408081056, + "loss": 2.464663111823029e-07, + "mean_token_accuracy": 0.6699574515223503, + "num_tokens": 8745995.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.108328685164452, + "rewards/margins": 19.001902163028717, + "rewards/rejected": -14.893573427200318, + "step": 950 + }, + { + "entropy": 1.2638258799910544, + "epoch": 2.8414814814814813, + "grad_norm": 2.4437904357910156e-06, + "learning_rate": 4.312756738160145e-07, + "logits/chosen": 2.2900851054697773, + "logits/rejected": -1.1548115943695305, + "logps/chosen": -62.25515162944794, + "logps/rejected": -332.41545429229734, + "loss": 1.340860762866214e-07, + "mean_token_accuracy": 0.6710602834820747, + "num_tokens": 8840853.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.389033368229866, + "rewards/margins": 19.693468022346497, + "rewards/rejected": -15.304434645175935, + "step": 960 + }, + { + "entropy": 1.2863718561828137, + "epoch": 2.871111111111111, + "grad_norm": 1.8358230590820312e-05, + "learning_rate": 2.8661166316229225e-07, + "logits/chosen": 2.3169417969141524, + "logits/rejected": -1.186848696642711, + "logps/chosen": -62.72764225006104, + "logps/rejected": -326.7688617706299, + "loss": 1.440903929506021e-07, + "mean_token_accuracy": 0.6690138332545758, + "num_tokens": 8932082.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.227344951033592, + "rewards/margins": 19.289413845539094, + "rewards/rejected": -15.062068808078767, + "step": 970 + }, + { + "entropy": 1.2781345948576928, + "epoch": 2.9007407407407406, + "grad_norm": 4.887580871582031e-06, + "learning_rate": 1.7127004595681729e-07, + "logits/chosen": 2.351716086708728, + "logits/rejected": -1.1749225494133453, + "logps/chosen": -69.60287075042724, + "logps/rejected": -329.7513824462891, + "loss": 1.1806017710114248e-07, + "mean_token_accuracy": 0.6486908756196499, + "num_tokens": 9029000.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.309024539589882, + "rewards/margins": 19.48654924631119, + "rewards/rejected": -15.177524662017822, + "step": 980 + }, + { + "entropy": 1.2663434036076069, + "epoch": 2.9303703703703703, + "grad_norm": 7.063150405883789e-06, + "learning_rate": 8.538767483325383e-08, + "logits/chosen": 2.3055120496617296, + "logits/rejected": -1.0696752761558757, + "logps/chosen": -65.15313308238983, + "logps/rejected": -333.3471397399902, + "loss": 1.6038972034948528e-07, + "mean_token_accuracy": 0.6714354790747166, + "num_tokens": 9120849.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.179977935552597, + "rewards/margins": 19.668525290489196, + "rewards/rejected": -15.488547360897064, + "step": 990 + }, + { + "entropy": 1.279862828552723, + "epoch": 2.96, + "grad_norm": 3.2633543014526367e-06, + "learning_rate": 2.9066449079634406e-08, + "logits/chosen": 2.3262460395698, + "logits/rejected": -1.0832974043675088, + "logps/chosen": -65.35808026790619, + "logps/rejected": -333.4561340332031, + "loss": 1.2052357760694577e-07, + "mean_token_accuracy": 0.654293454438448, + "num_tokens": 9214314.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.2864476755261425, + "rewards/margins": 19.81372677087784, + "rewards/rejected": -15.527279019355774, + "step": 1000 + }, + { + "epoch": 2.96, + "eval_entropy": 1.280155298593161, + "eval_logits/chosen": 2.2437926653321396, + "eval_logits/rejected": -1.1593534352886283, + "eval_logps/chosen": -68.00064637777689, + "eval_logps/rejected": -326.6981409246271, + "eval_loss": 6.490902251243824e-06, + "eval_mean_token_accuracy": 0.6637652328797987, + "eval_num_tokens": 9214314.0, + "eval_rewards/accuracies": 1.0, + "eval_rewards/chosen": 4.315061373310489, + "eval_rewards/margins": 19.479887595543495, + "eval_rewards/rejected": -15.164826126365394, + "eval_runtime": 39.7099, + "eval_samples_per_second": 7.177, + "eval_steps_per_second": 3.601, + "step": 1000 + } + ], + "logging_steps": 10, + "max_steps": 1013, + "num_input_tokens_seen": 0, + "num_train_epochs": 3, + "save_steps": 100, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 4.87406059521024e+17, + "train_batch_size": 2, + "trial_name": null, + "trial_params": null +} diff --git a/checkpoint-1000/training_args.bin b/checkpoint-1000/training_args.bin new file mode 100644 index 0000000..5ac503c --- /dev/null +++ b/checkpoint-1000/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9511438415c2456684b2907b1d3a2888ce92542e75957d8ee17f0b9dda11c01a +size 7249 diff --git a/checkpoint-1013/chat_template.jinja b/checkpoint-1013/chat_template.jinja new file mode 100644 index 0000000..28028c0 --- /dev/null +++ b/checkpoint-1013/chat_template.jinja @@ -0,0 +1,54 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0]['role'] == 'system' %} + {{- messages[0]['content'] }} + {%- else %} + {{- 'You are a helpful assistant.' }} + {%- endif %} + {{- "\n\n# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0]['role'] == 'system' %} + {{- '<|im_start|>system\n' + messages[0]['content'] + '<|im_end|>\n' }} + {%- else %} + {{- '<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- for message in messages %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) or (message.role == "assistant" and not message.tool_calls) %} + {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {{- '<|im_start|>' + message.role }} + {%- if message.content %} + {{- '\n' + message.content }} + {%- endif %} + {%- for tool_call in message.tool_calls %} + {%- if tool_call.function is defined %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {{- tool_call.arguments | tojson }} + {{- '}\n' }} + {%- endfor %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- message.content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} +{%- endif %} diff --git a/checkpoint-1013/config.json b/checkpoint-1013/config.json new file mode 100644 index 0000000..6b475aa --- /dev/null +++ b/checkpoint-1013/config.json @@ -0,0 +1,62 @@ +{ + "architectures": [ + "Qwen2ForCausalLM" + ], + "attention_dropout": 0.0, + "bos_token_id": null, + "dtype": "bfloat16", + "eos_token_id": 151643, + "hidden_act": "silu", + "hidden_size": 3584, + "initializer_range": 0.02, + "intermediate_size": 18944, + "layer_types": [ + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention" + ], + "max_position_embeddings": 131072, + "max_window_layers": 28, + "model_type": "qwen2", + "num_attention_heads": 28, + "num_hidden_layers": 28, + "num_key_value_heads": 4, + "pad_token_id": 151643, + "rms_norm_eps": 1e-06, + "rope_parameters": { + "rope_theta": 1000000.0, + "rope_type": "default" + }, + "sliding_window": null, + "tie_word_embeddings": false, + "transformers_version": "5.5.4", + "use_cache": false, + "use_mrope": false, + "use_sliding_window": false, + "vocab_size": 152064 +} diff --git a/checkpoint-1013/generation_config.json b/checkpoint-1013/generation_config.json new file mode 100644 index 0000000..0fb69a0 --- /dev/null +++ b/checkpoint-1013/generation_config.json @@ -0,0 +1,9 @@ +{ + "do_sample": true, + "eos_token_id": [ + 151643 + ], + "max_new_tokens": 2048, + "pad_token_id": 151643, + "transformers_version": "5.5.4" +} diff --git a/checkpoint-1013/model.safetensors b/checkpoint-1013/model.safetensors new file mode 100644 index 0000000..31cbc63 --- /dev/null +++ b/checkpoint-1013/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d028065a729c06acddc98ebf590676e754be525a368c5d864d15b420aeefb255 +size 15231272152 diff --git a/checkpoint-1013/optimizer.pt b/checkpoint-1013/optimizer.pt new file mode 100644 index 0000000..2bd26af --- /dev/null +++ b/checkpoint-1013/optimizer.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:06099e2c9c35cd3a066e2a6f9f56ecfc42d7f650ba704f026b10f8a5210f0d13 +size 18724493813 diff --git a/checkpoint-1013/rng_state.pth b/checkpoint-1013/rng_state.pth new file mode 100644 index 0000000..d2b156f --- /dev/null +++ b/checkpoint-1013/rng_state.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4801416a03beb0f63c300670679d3fb6cca48da8259362e9be9e6ffae0c5ffd0 +size 14645 diff --git a/checkpoint-1013/scheduler.pt b/checkpoint-1013/scheduler.pt new file mode 100644 index 0000000..f45e23e --- /dev/null +++ b/checkpoint-1013/scheduler.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:aa958ee472007d4d112924e9ae29b0fb29969e500ba3e7883b9d95b323ad597a +size 1465 diff --git a/checkpoint-1013/tokenizer.json b/checkpoint-1013/tokenizer.json new file mode 100644 index 0000000..34510ff --- /dev/null +++ b/checkpoint-1013/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3fd169731d2cbde95e10bf356d66d5997fd885dd8dbb6fb4684da3f23b2585d8 +size 11421892 diff --git a/checkpoint-1013/tokenizer_config.json b/checkpoint-1013/tokenizer_config.json new file mode 100644 index 0000000..c960ecf --- /dev/null +++ b/checkpoint-1013/tokenizer_config.json @@ -0,0 +1,29 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/checkpoint-1013/trainer_state.json b/checkpoint-1013/trainer_state.json new file mode 100644 index 0000000..9cb1a7e --- /dev/null +++ b/checkpoint-1013/trainer_state.json @@ -0,0 +1,2098 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 2.9985185185185186, + "eval_steps": 100, + "global_step": 1013, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0, + "eval_entropy": 1.7289148477407603, + "eval_logits/chosen": 0.8636333912138532, + "eval_logits/rejected": -0.6579137279251289, + "eval_logps/chosen": -111.15125904216633, + "eval_logps/rejected": -175.04988199514108, + "eval_loss": 0.6931471228599548, + "eval_mean_token_accuracy": 0.4934375546612106, + "eval_num_tokens": 0.0, + "eval_rewards/accuracies": 0.0, + "eval_rewards/chosen": 0.0, + "eval_rewards/margins": 0.0, + "eval_rewards/rejected": 0.0, + "eval_runtime": 39.0805, + "eval_samples_per_second": 7.293, + "eval_steps_per_second": 3.659, + "step": 0 + }, + { + "entropy": 1.7504533976316452, + "epoch": 0.002962962962962963, + "grad_norm": 83.5, + "learning_rate": 0.0, + "logits/chosen": 1.08207520218349, + "logits/rejected": -0.5940652974601717, + "logps/chosen": -110.63330745697021, + "logps/rejected": -182.49396514892578, + "loss": 0.6931471824645996, + "mean_token_accuracy": 0.5048413500189781, + "num_tokens": 10169.0, + "rewards/accuracies": 0.0, + "rewards/chosen": 0.0, + "rewards/margins": 0.0, + "rewards/rejected": 0.0, + "step": 1 + }, + { + "entropy": 1.7299097461832895, + "epoch": 0.02962962962962963, + "grad_norm": 29.875, + "learning_rate": 4.455445544554456e-06, + "logits/chosen": 0.8212555533910425, + "logits/rejected": -0.6693587809238846, + "logps/chosen": -100.81309244367812, + "logps/rejected": -171.8307899898953, + "loss": 0.5316412713792589, + "mean_token_accuracy": 0.5100594382319186, + "num_tokens": 89122.0, + "rewards/accuracies": 0.8125, + "rewards/chosen": 0.306247040655257, + "rewards/margins": 0.41913801974280634, + "rewards/rejected": -0.11289098787690616, + "step": 10 + }, + { + "entropy": 1.5854810774326324, + "epoch": 0.05925925925925926, + "grad_norm": 0.07470703125, + "learning_rate": 9.405940594059407e-06, + "logits/chosen": 1.2175614897456257, + "logits/rejected": -0.8402934758508309, + "logps/chosen": -77.16593537330627, + "logps/rejected": -196.31047840118407, + "loss": 0.036623942852020266, + "mean_token_accuracy": 0.5996901635080576, + "num_tokens": 182773.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 2.910937874764204, + "rewards/margins": 5.368858252465725, + "rewards/rejected": -2.4579203579574824, + "step": 20 + }, + { + "entropy": 1.4120451480150222, + "epoch": 0.08888888888888889, + "grad_norm": 0.019287109375, + "learning_rate": 1.4356435643564355e-05, + "logits/chosen": 2.264547569453187, + "logits/rejected": -1.0346384454567032, + "logps/chosen": -64.55358953475952, + "logps/rejected": -259.64974784851074, + "loss": 6.736992509104311e-05, + "mean_token_accuracy": 0.6582182548940182, + "num_tokens": 274816.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.462539532780648, + "rewards/margins": 12.943859684467316, + "rewards/rejected": -8.481320106983185, + "step": 30 + }, + { + "entropy": 1.355741088092327, + "epoch": 0.11851851851851852, + "grad_norm": 0.0004863739013671875, + "learning_rate": 1.930693069306931e-05, + "logits/chosen": 2.3368093516258224, + "logits/rejected": -1.142538018090554, + "logps/chosen": -61.829122233390805, + "logps/rejected": -295.9105268478394, + "loss": 3.1130424758885055e-06, + "mean_token_accuracy": 0.6669795714318753, + "num_tokens": 367268.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.366855585575104, + "rewards/margins": 16.085866248607637, + "rewards/rejected": -11.719010663032531, + "step": 40 + }, + { + "entropy": 1.320019119977951, + "epoch": 0.14814814814814814, + "grad_norm": 0.0001201629638671875, + "learning_rate": 2.4257425742574257e-05, + "logits/chosen": 2.315207261423255, + "logits/rejected": -1.0571934877438707, + "logps/chosen": -63.68361803293228, + "logps/rejected": -302.812979888916, + "loss": 7.578286749776453e-06, + "mean_token_accuracy": 0.6602454461157322, + "num_tokens": 461524.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.11230229139328, + "rewards/margins": 16.546387827396394, + "rewards/rejected": -12.43408569097519, + "step": 50 + }, + { + "entropy": 1.326953475177288, + "epoch": 0.17777777777777778, + "grad_norm": 0.0003871917724609375, + "learning_rate": 2.9207920792079208e-05, + "logits/chosen": 2.2619415245350254, + "logits/rejected": -1.0863394894289136, + "logps/chosen": -63.424767637252806, + "logps/rejected": -304.6159143447876, + "loss": 2.0567724277498202e-06, + "mean_token_accuracy": 0.6664863616228104, + "num_tokens": 555169.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.521066862344742, + "rewards/margins": 17.257024788856505, + "rewards/rejected": -12.735957789421082, + "step": 60 + }, + { + "entropy": 1.3109250903129577, + "epoch": 0.2074074074074074, + "grad_norm": 1.0609626770019531e-05, + "learning_rate": 3.415841584158416e-05, + "logits/chosen": 2.3168210798300413, + "logits/rejected": -1.1463735020520454, + "logps/chosen": -62.93964213132858, + "logps/rejected": -301.49362087249756, + "loss": 2.6633450761437417e-05, + "mean_token_accuracy": 0.6684455543756485, + "num_tokens": 646219.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.369684019684792, + "rewards/margins": 17.029080402851104, + "rewards/rejected": -12.659396553039551, + "step": 70 + }, + { + "entropy": 1.3183406174182892, + "epoch": 0.23703703703703705, + "grad_norm": 8.463859558105469e-06, + "learning_rate": 3.910891089108911e-05, + "logits/chosen": 2.2528845473791295, + "logits/rejected": -1.1451393864081993, + "logps/chosen": -64.70127091407775, + "logps/rejected": -313.31810626983645, + "loss": 2.219452881035977e-07, + "mean_token_accuracy": 0.6682900048792362, + "num_tokens": 741395.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.620437514781952, + "rewards/margins": 18.064030134677886, + "rewards/rejected": -13.443592536449433, + "step": 80 + }, + { + "entropy": 1.306735871732235, + "epoch": 0.26666666666666666, + "grad_norm": 0.00014209747314453125, + "learning_rate": 4.405940594059406e-05, + "logits/chosen": 2.2645358825214656, + "logits/rejected": -1.1447062023763102, + "logps/chosen": -62.47025227546692, + "logps/rejected": -309.65871505737306, + "loss": 6.8541798100341115e-06, + "mean_token_accuracy": 0.6588799960911274, + "num_tokens": 833512.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.536789815127849, + "rewards/margins": 17.808473789691924, + "rewards/rejected": -13.271684014797211, + "step": 90 + }, + { + "entropy": 1.3041012182831764, + "epoch": 0.2962962962962963, + "grad_norm": 0.0001277923583984375, + "learning_rate": 4.9009900990099014e-05, + "logits/chosen": 2.3280876455235577, + "logits/rejected": -1.0593703708195146, + "logps/chosen": -57.693548345565794, + "logps/rejected": -307.1459705352783, + "loss": 2.0000723452540113e-06, + "mean_token_accuracy": 0.6723956800997257, + "num_tokens": 925332.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.472308248281479, + "rewards/margins": 17.747703313827515, + "rewards/rejected": -13.275395059585572, + "step": 100 + }, + { + "epoch": 0.2962962962962963, + "eval_entropy": 1.3091890520149179, + "eval_logits/chosen": 2.2560516885971, + "eval_logits/rejected": -1.1380674279781138, + "eval_logps/chosen": -66.05370368157233, + "eval_logps/rejected": -310.2874222335282, + "eval_loss": 3.520939208101481e-05, + "eval_mean_token_accuracy": 0.662857598358101, + "eval_num_tokens": 925332.0, + "eval_rewards/accuracies": 1.0, + "eval_rewards/chosen": 4.5097555984983915, + "eval_rewards/margins": 18.033509881346376, + "eval_rewards/rejected": -13.523754186563558, + "eval_runtime": 38.3237, + "eval_samples_per_second": 7.437, + "eval_steps_per_second": 3.731, + "step": 100 + }, + { + "entropy": 1.284974904358387, + "epoch": 0.32592592592592595, + "grad_norm": 0.000812530517578125, + "learning_rate": 4.999050767562379e-05, + "logits/chosen": 2.3396127391184693, + "logits/rejected": -1.1138358422485608, + "logps/chosen": -61.88268647193909, + "logps/rejected": -304.68117294311526, + "loss": 3.422158624744043e-06, + "mean_token_accuracy": 0.6786789156496524, + "num_tokens": 1016416.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.462823995947838, + "rewards/margins": 17.767807400226594, + "rewards/rejected": -13.304983520507813, + "step": 110 + }, + { + "entropy": 1.2844461277127266, + "epoch": 0.35555555555555557, + "grad_norm": 0.00010204315185546875, + "learning_rate": 4.9951957461646706e-05, + "logits/chosen": 2.3448500846553584, + "logits/rejected": -1.0939366953296292, + "logps/chosen": -61.62471632957458, + "logps/rejected": -307.099747467041, + "loss": 1.316130146733485e-06, + "mean_token_accuracy": 0.6710287682712078, + "num_tokens": 1105772.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.635418501496315, + "rewards/margins": 18.310668289661407, + "rewards/rejected": -13.675249755382538, + "step": 120 + }, + { + "entropy": 1.2960298083722592, + "epoch": 0.3851851851851852, + "grad_norm": 0.0009765625, + "learning_rate": 4.988380179235843e-05, + "logits/chosen": 2.329744072704608, + "logits/rejected": -1.0829270884851314, + "logps/chosen": -60.90654925107956, + "logps/rejected": -320.0984771728516, + "loss": 2.281175875396002e-06, + "mean_token_accuracy": 0.6639963097870349, + "num_tokens": 1198464.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.473316392302513, + "rewards/margins": 18.291567063331605, + "rewards/rejected": -13.818250668048858, + "step": 130 + }, + { + "entropy": 1.2743454918265342, + "epoch": 0.4148148148148148, + "grad_norm": 0.00115966796875, + "learning_rate": 4.9786121534345265e-05, + "logits/chosen": 2.310538935492521, + "logits/rejected": -1.1208351833476473, + "logps/chosen": -58.558825969696045, + "logps/rejected": -314.1108278274536, + "loss": 1.2155161130067428e-06, + "mean_token_accuracy": 0.674963940680027, + "num_tokens": 1289352.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.255219912528991, + "rewards/margins": 18.211954617500304, + "rewards/rejected": -13.956734585762025, + "step": 140 + }, + { + "entropy": 1.2854739636182786, + "epoch": 0.4444444444444444, + "grad_norm": 0.0001659393310546875, + "learning_rate": 4.965903258506806e-05, + "logits/chosen": 2.33276765322688, + "logits/rejected": -1.1813850704996935, + "logps/chosen": -60.029894495010375, + "logps/rejected": -315.8014057159424, + "loss": 4.05059563490795e-07, + "mean_token_accuracy": 0.6699102215468884, + "num_tokens": 1380079.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.356227645277977, + "rewards/margins": 18.526466822624208, + "rewards/rejected": -14.170239269733429, + "step": 150 + }, + { + "entropy": 1.3013202093541623, + "epoch": 0.4740740740740741, + "grad_norm": 1.0132789611816406e-05, + "learning_rate": 4.950268573535012e-05, + "logits/chosen": 2.3245691177602428, + "logits/rejected": -1.0899626289102018, + "logps/chosen": -62.87379167079926, + "logps/rejected": -325.68464183807373, + "loss": 2.103474116665893e-07, + "mean_token_accuracy": 0.6679917454719544, + "num_tokens": 1474262.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.370475701987743, + "rewards/margins": 18.631532168388368, + "rewards/rejected": -14.261056447029114, + "step": 160 + }, + { + "entropy": 1.2932921536266804, + "epoch": 0.5037037037037037, + "grad_norm": 0.00017070770263671875, + "learning_rate": 4.93172664904641e-05, + "logits/chosen": 2.199208436926322, + "logits/rejected": -1.1313941206494675, + "logps/chosen": -66.81836426258087, + "logps/rejected": -313.0147771835327, + "loss": 5.84564577366109e-07, + "mean_token_accuracy": 0.6591029770672321, + "num_tokens": 1567351.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.2477530300617214, + "rewards/margins": 18.021551287174226, + "rewards/rejected": -13.773798143863678, + "step": 170 + }, + { + "entropy": 1.2906752578914165, + "epoch": 0.5333333333333333, + "grad_norm": 1.1622905731201172e-05, + "learning_rate": 4.9102994850030344e-05, + "logits/chosen": 2.257452431422654, + "logits/rejected": -1.1859710789627558, + "logps/chosen": -60.773719620704654, + "logps/rejected": -319.68780155181884, + "loss": 4.461732260097051e-07, + "mean_token_accuracy": 0.6724127769470215, + "num_tokens": 1659070.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.278084766864777, + "rewards/margins": 18.284436917304994, + "rewards/rejected": -14.006352269649506, + "step": 180 + }, + { + "entropy": 1.3018329843878746, + "epoch": 0.562962962962963, + "grad_norm": 6.4849853515625e-05, + "learning_rate": 4.886012504698769e-05, + "logits/chosen": 2.301720358967305, + "logits/rejected": -1.0489440936301595, + "logps/chosen": -61.81112604141235, + "logps/rejected": -315.3358419418335, + "loss": 5.486400823428994e-07, + "mean_token_accuracy": 0.669282128661871, + "num_tokens": 1750181.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.455310028791428, + "rewards/margins": 18.440570056438446, + "rewards/rejected": -13.985260021686553, + "step": 190 + }, + { + "entropy": 1.3009084612131119, + "epoch": 0.5925925925925926, + "grad_norm": 0.0087890625, + "learning_rate": 4.858894524594653e-05, + "logits/chosen": 2.2154390003090785, + "logits/rejected": -1.1232223850457532, + "logps/chosen": -66.61707370281219, + "logps/rejected": -316.3582260131836, + "loss": 5.881736069568433e-06, + "mean_token_accuracy": 0.6558836951851845, + "num_tokens": 1841453.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.423358170688152, + "rewards/margins": 18.573253047466277, + "rewards/rejected": -14.149895036220551, + "step": 200 + }, + { + "epoch": 0.5925925925925926, + "eval_entropy": 1.294411522108358, + "eval_logits/chosen": 2.25081917507982, + "eval_logits/rejected": -1.1530030385301466, + "eval_logps/chosen": -66.83867399842589, + "eval_logps/rejected": -318.13998082301, + "eval_loss": 1.2471568879846018e-05, + "eval_mean_token_accuracy": 0.6633061449844521, + "eval_num_tokens": 1841453.0, + "eval_rewards/accuracies": 1.0, + "eval_rewards/chosen": 4.431258590904982, + "eval_rewards/margins": 18.740268787304004, + "eval_rewards/rejected": -14.309010145547507, + "eval_runtime": 38.3878, + "eval_samples_per_second": 7.424, + "eval_steps_per_second": 3.725, + "step": 200 + }, + { + "entropy": 1.2944884866476059, + "epoch": 0.6222222222222222, + "grad_norm": 5.125999450683594e-05, + "learning_rate": 4.8289777201281974e-05, + "logits/chosen": 2.3454347980352996, + "logits/rejected": -1.1855823049416294, + "logps/chosen": -64.12132394313812, + "logps/rejected": -325.72925930023194, + "loss": 2.409943363090861e-07, + "mean_token_accuracy": 0.6670842688530684, + "num_tokens": 1935659.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.459967152774334, + "rewards/margins": 19.128938269615173, + "rewards/rejected": -14.668970954418182, + "step": 210 + }, + { + "entropy": 1.2553015358746051, + "epoch": 0.6518518518518519, + "grad_norm": 4.76837158203125e-05, + "learning_rate": 4.796297587537286e-05, + "logits/chosen": 2.308509979398422, + "logits/rejected": -1.0750038709449357, + "logps/chosen": -58.983665919303895, + "logps/rejected": -315.6023241043091, + "loss": 3.415452965782606e-07, + "mean_token_accuracy": 0.6794103197753429, + "num_tokens": 2029268.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.352584506571293, + "rewards/margins": 18.749567639827728, + "rewards/rejected": -14.396983158588409, + "step": 220 + }, + { + "entropy": 1.2749639593064785, + "epoch": 0.6814814814814815, + "grad_norm": 2.7894973754882812e-05, + "learning_rate": 4.760892901743944e-05, + "logits/chosen": 2.263447201357434, + "logits/rejected": -1.0746111454871972, + "logps/chosen": -64.98015263080597, + "logps/rejected": -320.59440212249757, + "loss": 1.3830200259690174e-06, + "mean_token_accuracy": 0.6660085044801235, + "num_tokens": 2123260.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.317273432016373, + "rewards/margins": 18.91294218301773, + "rewards/rejected": -14.59566867351532, + "step": 230 + }, + { + "entropy": 1.2890466079115868, + "epoch": 0.7111111111111111, + "grad_norm": 2.7894973754882812e-05, + "learning_rate": 4.722805670347963e-05, + "logits/chosen": 2.3375335604609835, + "logits/rejected": -1.1522054103145247, + "logps/chosen": -62.949660015106204, + "logps/rejected": -325.21918926239016, + "loss": 1.8574786508906982e-07, + "mean_token_accuracy": 0.6677135609090328, + "num_tokens": 2211888.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.302155223488808, + "rewards/margins": 19.20953975915909, + "rewards/rejected": -14.90738466978073, + "step": 240 + }, + { + "entropy": 1.297672052681446, + "epoch": 0.7407407407407407, + "grad_norm": 1.3649463653564453e-05, + "learning_rate": 4.6820810837849536e-05, + "logits/chosen": 2.3567186399215143, + "logits/rejected": -1.2153430976155177, + "logps/chosen": -61.082769107818606, + "logps/rejected": -327.5022741317749, + "loss": 4.804733634955483e-07, + "mean_token_accuracy": 0.6871423445641994, + "num_tokens": 2307633.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.415084969997406, + "rewards/margins": 19.260318982601166, + "rewards/rejected": -14.845234000682831, + "step": 250 + }, + { + "entropy": 1.289575246721506, + "epoch": 0.7703703703703704, + "grad_norm": 0.00013637542724609375, + "learning_rate": 4.63876746170797e-05, + "logits/chosen": 2.2677862381437905, + "logits/rejected": -1.1125468158062497, + "logps/chosen": -67.2604984998703, + "logps/rejected": -321.5008083343506, + "loss": 1.5667228581150994e-07, + "mean_token_accuracy": 0.6551826104521752, + "num_tokens": 2404339.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.320639500021935, + "rewards/margins": 18.89977525472641, + "rewards/rejected": -14.57913579940796, + "step": 260 + }, + { + "entropy": 1.2907186925411225, + "epoch": 0.8, + "grad_norm": 2.5987625122070312e-05, + "learning_rate": 4.592916195656322e-05, + "logits/chosen": 2.337716368388003, + "logits/rejected": -1.1807755553775976, + "logps/chosen": -61.246209836006166, + "logps/rejected": -321.838525390625, + "loss": 3.219545988031314e-07, + "mean_token_accuracy": 0.67101164534688, + "num_tokens": 2493582.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.55239104628563, + "rewards/margins": 19.164004135131837, + "rewards/rejected": -14.611613166332244, + "step": 270 + }, + { + "entropy": 1.2973388336598872, + "epoch": 0.8296296296296296, + "grad_norm": 1.0728836059570312e-05, + "learning_rate": 4.544581688079602e-05, + "logits/chosen": 2.2622078397191254, + "logits/rejected": -1.1987081179562755, + "logps/chosen": -62.545769691467285, + "logps/rejected": -328.43379402160645, + "loss": 1.6840259604578022e-07, + "mean_token_accuracy": 0.6727346412837505, + "num_tokens": 2583501.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.605609413981438, + "rewards/margins": 19.350289702415466, + "rewards/rejected": -14.744680321216583, + "step": 280 + }, + { + "entropy": 1.270925548672676, + "epoch": 0.8592592592592593, + "grad_norm": 1.4722347259521484e-05, + "learning_rate": 4.4938212877892725e-05, + "logits/chosen": 2.304147609404456, + "logits/rejected": -1.1603299402866345, + "logps/chosen": -64.19616587162018, + "logps/rejected": -321.61998023986814, + "loss": 1.092423531190434e-07, + "mean_token_accuracy": 0.6694903835654259, + "num_tokens": 2673092.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.521212270855903, + "rewards/margins": 19.481856608390807, + "rewards/rejected": -14.960644149780274, + "step": 290 + }, + { + "entropy": 1.2587753705680371, + "epoch": 0.8888888888888888, + "grad_norm": 8.046627044677734e-06, + "learning_rate": 4.4406952219143936e-05, + "logits/chosen": 2.2447284227823454, + "logits/rejected": -1.2047798430182726, + "logps/chosen": -63.220883178710935, + "logps/rejected": -316.7784074783325, + "loss": 9.590096851752606e-07, + "mean_token_accuracy": 0.6734147310256958, + "num_tokens": 2765264.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.453023025393486, + "rewards/margins": 19.15658880472183, + "rewards/rejected": -14.703565871715545, + "step": 300 + }, + { + "epoch": 0.8888888888888888, + "eval_entropy": 1.2864730945833913, + "eval_logits/chosen": 2.245142647914562, + "eval_logits/rejected": -1.1577701959581186, + "eval_logps/chosen": -67.51500383123651, + "eval_logps/rejected": -323.1507350681545, + "eval_loss": 9.1151678134338e-06, + "eval_mean_token_accuracy": 0.6627988360978506, + "eval_num_tokens": 2765264.0, + "eval_rewards/accuracies": 1.0, + "eval_rewards/chosen": 4.3636256072904684, + "eval_rewards/margins": 19.173711116497334, + "eval_rewards/rejected": -14.810085516709547, + "eval_runtime": 38.408, + "eval_samples_per_second": 7.42, + "eval_steps_per_second": 3.723, + "step": 300 + }, + { + "entropy": 1.2768845409154892, + "epoch": 0.9185185185185185, + "grad_norm": 4.315376281738281e-05, + "learning_rate": 4.385266524442241e-05, + "logits/chosen": 2.2955878654382667, + "logits/rejected": -1.200506409819956, + "logps/chosen": -62.47047922611237, + "logps/rejected": -329.1678291320801, + "loss": 2.0756017420353602e-07, + "mean_token_accuracy": 0.6736313477158546, + "num_tokens": 2860290.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.335641312599182, + "rewards/margins": 19.35282862186432, + "rewards/rejected": -15.01718727350235, + "step": 310 + }, + { + "entropy": 1.267455793917179, + "epoch": 0.9481481481481482, + "grad_norm": 2.2649765014648438e-05, + "learning_rate": 4.3276009614285825e-05, + "logits/chosen": 2.301217628396178, + "logits/rejected": -1.1612455940414776, + "logps/chosen": -60.544256925582886, + "logps/rejected": -325.25582218170166, + "loss": 8.637363748675853e-08, + "mean_token_accuracy": 0.6695008479058743, + "num_tokens": 2951326.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.26083664894104, + "rewards/margins": 19.293893361091612, + "rewards/rejected": -15.033056676387787, + "step": 320 + }, + { + "entropy": 1.2802438855171203, + "epoch": 0.9777777777777777, + "grad_norm": 3.1441450119018555e-06, + "learning_rate": 4.267766952966369e-05, + "logits/chosen": 2.3043263280692856, + "logits/rejected": -1.1247859920954724, + "logps/chosen": -65.50963315963745, + "logps/rejected": -327.67705821990967, + "loss": 1.6503474398632533e-06, + "mean_token_accuracy": 0.6599100351333618, + "num_tokens": 3045520.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.30358607918024, + "rewards/margins": 19.168729376792907, + "rewards/rejected": -14.865143322944641, + "step": 330 + }, + { + "entropy": 1.2861412669482983, + "epoch": 1.005925925925926, + "grad_norm": 5.793571472167969e-05, + "learning_rate": 4.205835492005405e-05, + "logits/chosen": 2.319402775016898, + "logits/rejected": -1.2147055357233236, + "logps/chosen": -67.06965022338063, + "logps/rejected": -330.37220262226305, + "loss": 3.5803138871415284e-07, + "mean_token_accuracy": 0.6624507841311003, + "num_tokens": 3130709.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.281106482603048, + "rewards/margins": 19.5897334500363, + "rewards/rejected": -15.308626877634149, + "step": 340 + }, + { + "entropy": 1.2823633886873722, + "epoch": 1.0355555555555556, + "grad_norm": 6.008148193359375e-05, + "learning_rate": 4.141880060119336e-05, + "logits/chosen": 2.339569692570856, + "logits/rejected": -1.147382590046395, + "logps/chosen": -60.31965174674988, + "logps/rejected": -335.1733335494995, + "loss": 1.532446731289383e-07, + "mean_token_accuracy": 0.6735855638980865, + "num_tokens": 3225369.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.492888362705708, + "rewards/margins": 19.827855718135833, + "rewards/rejected": -15.334967303276063, + "step": 350 + }, + { + "entropy": 1.2730201914906503, + "epoch": 1.0651851851851852, + "grad_norm": 0.0001926422119140625, + "learning_rate": 4.075976540319888e-05, + "logits/chosen": 2.3003171439882366, + "logits/rejected": -1.2015335101578284, + "logps/chosen": -60.594508814811704, + "logps/rejected": -329.16415615081786, + "loss": 4.122233804082498e-07, + "mean_token_accuracy": 0.6743721053004265, + "num_tokens": 3318818.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.401582342386246, + "rewards/margins": 19.628432750701904, + "rewards/rejected": -15.22685045003891, + "step": 360 + }, + { + "entropy": 1.2773501306772232, + "epoch": 1.094814814814815, + "grad_norm": 0.000591278076171875, + "learning_rate": 4.0082031270217976e-05, + "logits/chosen": 2.2876778332841776, + "logits/rejected": -1.154664545777339, + "logps/chosen": -63.76930930614471, + "logps/rejected": -319.00229358673096, + "loss": 1.1508095667522866e-06, + "mean_token_accuracy": 0.6718519743531942, + "num_tokens": 3410676.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.197161988914013, + "rewards/margins": 19.0069660782814, + "rewards/rejected": -14.809803891181947, + "step": 370 + }, + { + "entropy": 1.2946350619196891, + "epoch": 1.1244444444444444, + "grad_norm": 0.00022411346435546875, + "learning_rate": 3.9386402332652754e-05, + "logits/chosen": 2.293869972488772, + "logits/rejected": -1.1689247316110847, + "logps/chosen": -64.16428699493409, + "logps/rejected": -339.3799625396729, + "loss": 1.2371945103950566e-07, + "mean_token_accuracy": 0.664111914485693, + "num_tokens": 3504356.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.358540344238281, + "rewards/margins": 19.867380928993224, + "rewards/rejected": -15.508840537071228, + "step": 380 + }, + { + "entropy": 1.2926758706569672, + "epoch": 1.154074074074074, + "grad_norm": 3.6954879760742188e-06, + "learning_rate": 3.867370395306068e-05, + "logits/chosen": 2.1766569425767703, + "logits/rejected": -1.087643152794101, + "logps/chosen": -69.47466747760772, + "logps/rejected": -327.71664810180664, + "loss": 1.194763967760082e-07, + "mean_token_accuracy": 0.6569934625178575, + "num_tokens": 3594396.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.461885306239128, + "rewards/margins": 19.697250258922576, + "rewards/rejected": -15.235364770889282, + "step": 390 + }, + { + "entropy": 1.2546571284532546, + "epoch": 1.1837037037037037, + "grad_norm": 8.940696716308594e-06, + "learning_rate": 3.794478174686328e-05, + "logits/chosen": 2.28344883152218, + "logits/rejected": -1.1421346511748105, + "logps/chosen": -64.66728138923645, + "logps/rejected": -324.4414520263672, + "loss": 7.843665343898465e-08, + "mean_token_accuracy": 0.662892397120595, + "num_tokens": 3682850.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.419300290942192, + "rewards/margins": 19.782128620147706, + "rewards/rejected": -15.362828314304352, + "step": 400 + }, + { + "epoch": 1.1837037037037037, + "eval_entropy": 1.2814960267160322, + "eval_logits/chosen": 2.2466129676027395, + "eval_logits/rejected": -1.158793695801587, + "eval_logps/chosen": -67.91167767398007, + "eval_logps/rejected": -326.0148432804988, + "eval_loss": 7.35781259209034e-06, + "eval_mean_token_accuracy": 0.6629326431067674, + "eval_num_tokens": 3682850.0, + "eval_rewards/accuracies": 1.0, + "eval_rewards/chosen": 4.323958210178189, + "eval_rewards/margins": 19.420454478764032, + "eval_rewards/rejected": -15.096496268585845, + "eval_runtime": 38.5811, + "eval_samples_per_second": 7.387, + "eval_steps_per_second": 3.706, + "step": 400 + }, + { + "entropy": 1.2794287726283073, + "epoch": 1.2133333333333334, + "grad_norm": 6.0498714447021484e-06, + "learning_rate": 3.720050057902496e-05, + "logits/chosen": 2.2348389102512294, + "logits/rejected": -1.20443931083074, + "logps/chosen": -61.86296122074127, + "logps/rejected": -331.40450420379636, + "loss": 1.5817265648365718e-07, + "mean_token_accuracy": 0.6723401784896851, + "num_tokens": 3776120.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.320593364536762, + "rewards/margins": 19.594025874137877, + "rewards/rejected": -15.273432505130767, + "step": 410 + }, + { + "entropy": 1.2816326960921287, + "epoch": 1.242962962962963, + "grad_norm": 2.7865171432495117e-06, + "learning_rate": 3.644174353789204e-05, + "logits/chosen": 2.2707540738720113, + "logits/rejected": -1.191927789160487, + "logps/chosen": -64.48058602809905, + "logps/rejected": -332.1292999267578, + "loss": 9.944698149411124e-08, + "mean_token_accuracy": 0.6633661832660437, + "num_tokens": 3868574.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.300156235694885, + "rewards/margins": 19.600781667232514, + "rewards/rejected": -15.30062551498413, + "step": 420 + }, + { + "entropy": 1.2759614169597626, + "epoch": 1.2725925925925927, + "grad_norm": 6.29425048828125e-05, + "learning_rate": 3.5669410887410095e-05, + "logits/chosen": 2.3054515861422304, + "logits/rejected": -1.1359497002203924, + "logps/chosen": -64.89167664051055, + "logps/rejected": -334.02518730163575, + "loss": 2.404902943453635e-07, + "mean_token_accuracy": 0.6750707887113094, + "num_tokens": 3961842.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.182991920411586, + "rewards/margins": 19.472245895862578, + "rewards/rejected": -15.289253902435302, + "step": 430 + }, + { + "entropy": 1.2799105122685432, + "epoch": 1.3022222222222222, + "grad_norm": 0.00012683868408203125, + "learning_rate": 3.488441899896217e-05, + "logits/chosen": 2.345937521884795, + "logits/rejected": -1.2320921962004756, + "logps/chosen": -58.358681523799895, + "logps/rejected": -320.94243869781496, + "loss": 2.9142654511815635e-07, + "mean_token_accuracy": 0.679117338731885, + "num_tokens": 4051000.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.320470458269119, + "rewards/margins": 19.39275724887848, + "rewards/rejected": -15.072286760807037, + "step": 440 + }, + { + "entropy": 1.2606690004467964, + "epoch": 1.3318518518518518, + "grad_norm": 3.933906555175781e-05, + "learning_rate": 3.4087699264095745e-05, + "logits/chosen": 2.3590029943979434, + "logits/rejected": -1.180752685047298, + "logps/chosen": -57.81958304643631, + "logps/rejected": -327.3783897399902, + "loss": 1.31027945826645e-07, + "mean_token_accuracy": 0.6829047620296478, + "num_tokens": 4144901.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.355239224433899, + "rewards/margins": 19.49090631008148, + "rewards/rejected": -15.135667026042938, + "step": 450 + }, + { + "entropy": 1.2720958605408668, + "epoch": 1.3614814814814815, + "grad_norm": 7.152557373046875e-06, + "learning_rate": 3.328019698942826e-05, + "logits/chosen": 2.28385155999331, + "logits/rejected": -1.0655708622856899, + "logps/chosen": -65.57811061143875, + "logps/rejected": -324.391535949707, + "loss": 1.4767925904379808e-07, + "mean_token_accuracy": 0.6687033347785473, + "num_tokens": 4235782.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.3411215707659725, + "rewards/margins": 19.290599715709686, + "rewards/rejected": -14.949478232860566, + "step": 460 + }, + { + "entropy": 1.2874604403972625, + "epoch": 1.3911111111111112, + "grad_norm": 0.00011920928955078125, + "learning_rate": 3.246287027504237e-05, + "logits/chosen": 2.2650682684340646, + "logits/rejected": -1.1018559007875284, + "logps/chosen": -63.989255738258365, + "logps/rejected": -333.0100872039795, + "loss": 2.971911953864037e-07, + "mean_token_accuracy": 0.6702111043035984, + "num_tokens": 4327602.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.187621220946312, + "rewards/margins": 19.38519823551178, + "rewards/rejected": -15.197577047348023, + "step": 470 + }, + { + "entropy": 1.293594116717577, + "epoch": 1.4207407407407406, + "grad_norm": 9.870529174804688e-05, + "learning_rate": 3.163668887770181e-05, + "logits/chosen": 2.306861956625707, + "logits/rejected": -1.1661940995045186, + "logps/chosen": -61.81727409362793, + "logps/rejected": -332.9074993133545, + "loss": 2.5863685095828257e-07, + "mean_token_accuracy": 0.6736553743481636, + "num_tokens": 4419888.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.17706933170557, + "rewards/margins": 19.331134915351868, + "rewards/rejected": -15.154065454006195, + "step": 480 + }, + { + "entropy": 1.2759695634245873, + "epoch": 1.4503703703703703, + "grad_norm": 6.771087646484375e-05, + "learning_rate": 3.08026330602367e-05, + "logits/chosen": 2.338572971372103, + "logits/rejected": -1.1794468686720447, + "logps/chosen": -58.53629996776581, + "logps/rejected": -332.7186403274536, + "loss": 8.574451726417465e-08, + "mean_token_accuracy": 0.6770360998809337, + "num_tokens": 4511414.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.380688160657883, + "rewards/margins": 19.57960090637207, + "rewards/rejected": -15.198912739753723, + "step": 490 + }, + { + "entropy": 1.2591664418578148, + "epoch": 1.48, + "grad_norm": 8.046627044677734e-06, + "learning_rate": 2.9961692428463285e-05, + "logits/chosen": 2.2509338882701817, + "logits/rejected": -1.136340958920119, + "logps/chosen": -66.00177557468415, + "logps/rejected": -332.10723934173586, + "loss": 3.2082580219139346e-07, + "mean_token_accuracy": 0.6653452910482883, + "num_tokens": 4604818.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.144159619510174, + "rewards/margins": 19.529365801811217, + "rewards/rejected": -15.385206091403962, + "step": 500 + }, + { + "epoch": 1.48, + "eval_entropy": 1.2810596311009013, + "eval_logits/chosen": 2.2449733761384874, + "eval_logits/rejected": -1.1597314363523379, + "eval_logps/chosen": -67.92375413854639, + "eval_logps/rejected": -326.1967938829969, + "eval_loss": 7.327445018745493e-06, + "eval_mean_token_accuracy": 0.6633227342492217, + "eval_num_tokens": 4604818.0, + "eval_rewards/accuracies": 1.0, + "eval_rewards/chosen": 4.322750580894364, + "eval_rewards/margins": 19.43744218599546, + "eval_rewards/rejected": -15.114691607601992, + "eval_runtime": 38.6636, + "eval_samples_per_second": 7.371, + "eval_steps_per_second": 3.699, + "step": 500 + }, + { + "entropy": 1.2616984777152538, + "epoch": 1.5096296296296297, + "grad_norm": 0.000125885009765625, + "learning_rate": 2.9114864757018352e-05, + "logits/chosen": 2.294192165543051, + "logits/rejected": -1.0944581394998258, + "logps/chosen": -64.26435973644257, + "logps/rejected": -325.8657173156738, + "loss": 1.0070654070659657e-07, + "mean_token_accuracy": 0.6660138607025147, + "num_tokens": 4695387.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.35933136343956, + "rewards/margins": 19.695837664604188, + "rewards/rejected": -15.336506283283233, + "step": 510 + }, + { + "entropy": 1.2706036016345024, + "epoch": 1.5392592592592593, + "grad_norm": 8.821487426757812e-05, + "learning_rate": 2.8263154805501297e-05, + "logits/chosen": 2.228209829000261, + "logits/rejected": -1.113362437458028, + "logps/chosen": -69.59165489673615, + "logps/rejected": -321.3457368850708, + "loss": 1.4221999435903854e-07, + "mean_token_accuracy": 0.6667736105620861, + "num_tokens": 4788711.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.314350298047065, + "rewards/margins": 19.27485271692276, + "rewards/rejected": -14.960502445697784, + "step": 520 + }, + { + "entropy": 1.2767572030425072, + "epoch": 1.568888888888889, + "grad_norm": 0.0002460479736328125, + "learning_rate": 2.7407573126328545e-05, + "logits/chosen": 2.3174187011624654, + "logits/rejected": -1.2211792118899774, + "logps/chosen": -58.480189538002016, + "logps/rejected": -321.1500442504883, + "loss": 1.383585185976699e-07, + "mean_token_accuracy": 0.675158528238535, + "num_tokens": 4877592.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.162171944975853, + "rewards/margins": 19.111147022247316, + "rewards/rejected": -14.948975014686585, + "step": 530 + }, + { + "entropy": 1.2621485769748688, + "epoch": 1.5985185185185187, + "grad_norm": 3.844499588012695e-06, + "learning_rate": 2.6549134865714868e-05, + "logits/chosen": 2.26315295877127, + "logits/rejected": -1.12053805020359, + "logps/chosen": -64.27606203556061, + "logps/rejected": -328.73465633392334, + "loss": 1.7509564713691362e-07, + "mean_token_accuracy": 0.670067022740841, + "num_tokens": 4969700.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.281642350554466, + "rewards/margins": 19.606130039691926, + "rewards/rejected": -15.32448751926422, + "step": 540 + }, + { + "entropy": 1.266157256066799, + "epoch": 1.6281481481481481, + "grad_norm": 1.4901161193847656e-05, + "learning_rate": 2.5688858559204053e-05, + "logits/chosen": 2.3036601533908074, + "logits/rejected": -1.1204950201201778, + "logps/chosen": -63.52790377140045, + "logps/rejected": -332.37768840789795, + "loss": 1.9515678104653488e-07, + "mean_token_accuracy": 0.6726867496967316, + "num_tokens": 5064766.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.242260693013668, + "rewards/margins": 19.574455058574678, + "rewards/rejected": -15.33219426870346, + "step": 550 + }, + { + "entropy": 1.272486585378647, + "epoch": 1.6577777777777778, + "grad_norm": 3.933906555175781e-06, + "learning_rate": 2.4827764923178246e-05, + "logits/chosen": 2.3149282625691305, + "logits/rejected": -1.2307484890552138, + "logps/chosen": -60.608607578277585, + "logps/rejected": -325.5159580230713, + "loss": 1.471338123337773e-07, + "mean_token_accuracy": 0.6735777668654919, + "num_tokens": 5155008.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.4290194630622866, + "rewards/margins": 19.601059877872466, + "rewards/rejected": -15.172040581703186, + "step": 560 + }, + { + "entropy": 1.2813369497656821, + "epoch": 1.6874074074074072, + "grad_norm": 0.000339508056640625, + "learning_rate": 2.3966875643779667e-05, + "logits/chosen": 2.2549751619675136, + "logits/rejected": -1.1594325055530381, + "logps/chosen": -64.34185855388641, + "logps/rejected": -327.56716537475586, + "loss": 1.8051871393254259e-07, + "mean_token_accuracy": 0.6600819997489452, + "num_tokens": 5248360.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.4127004653215405, + "rewards/margins": 19.473870050907134, + "rewards/rejected": -15.061169505119324, + "step": 570 + }, + { + "entropy": 1.2690633878111839, + "epoch": 1.717037037037037, + "grad_norm": 0.000118255615234375, + "learning_rate": 2.3107212164681776e-05, + "logits/chosen": 2.277724768747875, + "logits/rejected": -1.1787549020085861, + "logps/chosen": -66.66345942020416, + "logps/rejected": -323.8808866500855, + "loss": 2.298829940627911e-07, + "mean_token_accuracy": 0.6669816076755524, + "num_tokens": 5340554.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.2985071912407875, + "rewards/margins": 19.356661868095397, + "rewards/rejected": -15.058154690265656, + "step": 580 + }, + { + "entropy": 1.303315782546997, + "epoch": 1.7466666666666666, + "grad_norm": 9.894371032714844e-06, + "learning_rate": 2.224979447514802e-05, + "logits/chosen": 2.2417929976862743, + "logits/rejected": -1.1242295347690896, + "logps/chosen": -66.85722150802613, + "logps/rejected": -326.75849361419677, + "loss": 5.121967660670635e-07, + "mean_token_accuracy": 0.6579514071345329, + "num_tokens": 5433019.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.279269021749497, + "rewards/margins": 19.181958985328674, + "rewards/rejected": -14.902689921855927, + "step": 590 + }, + { + "entropy": 1.2511712044477463, + "epoch": 1.7762962962962963, + "grad_norm": 1.811981201171875e-05, + "learning_rate": 2.1395639899816333e-05, + "logits/chosen": 2.4288169740201644, + "logits/rejected": -1.0983362881453427, + "logps/chosen": -58.963488447666165, + "logps/rejected": -328.273935508728, + "loss": 5.187651368032676e-07, + "mean_token_accuracy": 0.6844911381602288, + "num_tokens": 5527564.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.607324048876762, + "rewards/margins": 19.77645412683487, + "rewards/rejected": -15.169130110740662, + "step": 600 + }, + { + "epoch": 1.7762962962962963, + "eval_entropy": 1.2805373877078503, + "eval_logits/chosen": 2.243851235815972, + "eval_logits/rejected": -1.1594670058021077, + "eval_logps/chosen": -67.99770445923706, + "eval_logps/rejected": -326.55811144421983, + "eval_loss": 6.923921773704933e-06, + "eval_mean_token_accuracy": 0.6634519100189209, + "eval_num_tokens": 5527564.0, + "eval_rewards/accuracies": 1.0, + "eval_rewards/chosen": 4.315355520148377, + "eval_rewards/margins": 19.466178794007202, + "eval_rewards/rejected": -15.15082325968709, + "eval_runtime": 39.0736, + "eval_samples_per_second": 7.294, + "eval_steps_per_second": 3.66, + "step": 600 + }, + { + "entropy": 1.2721578769385815, + "epoch": 1.805925925925926, + "grad_norm": 1.3053417205810547e-05, + "learning_rate": 2.054576189164518e-05, + "logits/chosen": 2.3202715445664515, + "logits/rejected": -1.132963813611814, + "logps/chosen": -60.805744123458865, + "logps/rejected": -332.585825920105, + "loss": 3.902006255884771e-08, + "mean_token_accuracy": 0.6726082235574722, + "num_tokens": 5620386.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.585019749403, + "rewards/margins": 20.017272901535033, + "rewards/rejected": -15.43225302696228, + "step": 610 + }, + { + "entropy": 1.2652241297066211, + "epoch": 1.8355555555555556, + "grad_norm": 1.633167266845703e-05, + "learning_rate": 1.9701168829453306e-05, + "logits/chosen": 2.259219360616968, + "logits/rejected": -1.0503303018419232, + "logps/chosen": -66.24489846229554, + "logps/rejected": -322.5876424789429, + "loss": 1.7130523701780476e-07, + "mean_token_accuracy": 0.6619290798902512, + "num_tokens": 5712637.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.140004143118858, + "rewards/margins": 19.201012241840363, + "rewards/rejected": -15.06100798845291, + "step": 620 + }, + { + "entropy": 1.2682488977909088, + "epoch": 1.8651851851851853, + "grad_norm": 2.956390380859375e-05, + "learning_rate": 1.8862862821480025e-05, + "logits/chosen": 2.3071657025101677, + "logits/rejected": -1.1965660990092328, + "logps/chosen": -65.7512573480606, + "logps/rejected": -326.63800010681155, + "loss": 9.80972572506289e-08, + "mean_token_accuracy": 0.6676984764635563, + "num_tokens": 5804212.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.233162444829941, + "rewards/margins": 19.35871180295944, + "rewards/rejected": -15.125549244880677, + "step": 630 + }, + { + "entropy": 1.2738682702183723, + "epoch": 1.894814814814815, + "grad_norm": 1.3709068298339844e-06, + "learning_rate": 1.8031838516385423e-05, + "logits/chosen": 2.277379836690332, + "logits/rejected": -1.1057443809996192, + "logps/chosen": -64.37387623786927, + "logps/rejected": -329.1884712219238, + "loss": 5.206376272326452e-08, + "mean_token_accuracy": 0.6642427556216717, + "num_tokens": 5897138.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.30352343916893, + "rewards/margins": 19.877286291122438, + "rewards/rejected": -15.573762822151185, + "step": 640 + }, + { + "entropy": 1.2760708816349506, + "epoch": 1.9244444444444444, + "grad_norm": 5.1021575927734375e-05, + "learning_rate": 1.7209081923101472e-05, + "logits/chosen": 2.327136815796652, + "logits/rejected": -1.1425043440165608, + "logps/chosen": -66.02759101390839, + "logps/rejected": -332.0861042022705, + "loss": 1.4513241239910712e-07, + "mean_token_accuracy": 0.66774348244071, + "num_tokens": 5990180.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.311979895830154, + "rewards/margins": 19.720207822322845, + "rewards/rejected": -15.408227932453155, + "step": 650 + }, + { + "entropy": 1.2696824759244918, + "epoch": 1.954074074074074, + "grad_norm": 2.4557113647460938e-05, + "learning_rate": 1.639556924093404e-05, + "logits/chosen": 2.2885721283972242, + "logits/rejected": -1.12765834632216, + "logps/chosen": -64.96237716674804, + "logps/rejected": -326.4824584960937, + "loss": 1.2898549357487353e-07, + "mean_token_accuracy": 0.6572523653507233, + "num_tokens": 6084186.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.2255510374903675, + "rewards/margins": 19.46041429042816, + "rewards/rejected": -15.234863257408142, + "step": 660 + }, + { + "entropy": 1.2641157731413841, + "epoch": 1.9837037037037037, + "grad_norm": 1.9073486328125e-06, + "learning_rate": 1.5592265701304114e-05, + "logits/chosen": 2.3036376802600183, + "logits/rejected": -1.1734117772095596, + "logps/chosen": -65.30954070091248, + "logps/rejected": -323.9220626831055, + "loss": 2.2752515178581233e-07, + "mean_token_accuracy": 0.6635996960103512, + "num_tokens": 6177151.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.288326196372509, + "rewards/margins": 19.406356739997864, + "rewards/rejected": -15.118030607700348, + "step": 670 + }, + { + "entropy": 1.2779945597836846, + "epoch": 2.011851851851852, + "grad_norm": 0.000255584716796875, + "learning_rate": 1.4800124422502334e-05, + "logits/chosen": 2.3972204176263543, + "logits/rejected": -1.1209583839266675, + "logps/chosen": -61.41229806448284, + "logps/rejected": -324.14299693860505, + "loss": 3.6133037610852626e-07, + "mean_token_accuracy": 0.6630792186448449, + "num_tokens": 6262744.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.139316172976243, + "rewards/margins": 19.215763267717865, + "rewards/rejected": -15.076447160620438, + "step": 680 + }, + { + "entropy": 1.266596408188343, + "epoch": 2.0414814814814815, + "grad_norm": 4.4889748096466064e-07, + "learning_rate": 1.4020085278815745e-05, + "logits/chosen": 2.3154754788175156, + "logits/rejected": -1.152066471886049, + "logps/chosen": -62.28092246055603, + "logps/rejected": -331.86610050201415, + "loss": 7.074050927258213e-08, + "mean_token_accuracy": 0.6724916785955429, + "num_tokens": 6357968.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.411373123526573, + "rewards/margins": 19.874622988700867, + "rewards/rejected": -15.4632497549057, + "step": 690 + }, + { + "entropy": 1.2671823538839817, + "epoch": 2.071111111111111, + "grad_norm": 0.0002994537353515625, + "learning_rate": 1.3253073785368547e-05, + "logits/chosen": 2.29187693793979, + "logits/rejected": -1.1263780908242413, + "logps/chosen": -65.86139830350876, + "logps/rejected": -331.38874130249025, + "loss": 2.163802719223895e-07, + "mean_token_accuracy": 0.6635798320174218, + "num_tokens": 6449816.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.283302050828934, + "rewards/margins": 19.748378670215608, + "rewards/rejected": -15.465076541900634, + "step": 700 + }, + { + "epoch": 2.071111111111111, + "eval_entropy": 1.2803548124286679, + "eval_logits/chosen": 2.243403260803776, + "eval_logits/rejected": -1.1587966303436696, + "eval_logps/chosen": -68.02522693980823, + "eval_logps/rejected": -326.6569073016827, + "eval_loss": 7.350444775511278e-06, + "eval_mean_token_accuracy": 0.6639135730016482, + "eval_num_tokens": 6449816.0, + "eval_rewards/accuracies": 1.0, + "eval_rewards/chosen": 4.312603289847607, + "eval_rewards/margins": 19.473306129028746, + "eval_rewards/rejected": -15.160702772073812, + "eval_runtime": 39.1309, + "eval_samples_per_second": 7.283, + "eval_steps_per_second": 3.654, + "step": 700 + }, + { + "entropy": 1.2728232815861702, + "epoch": 2.100740740740741, + "grad_norm": 8.821487426757812e-05, + "learning_rate": 1.2500000000000006e-05, + "logits/chosen": 2.255679331290835, + "logits/rejected": -1.1216202176026864, + "logps/chosen": -61.825205373764035, + "logps/rejected": -323.78655910491943, + "loss": 2.141799996024929e-07, + "mean_token_accuracy": 0.667589108645916, + "num_tokens": 6540123.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.437801697850228, + "rewards/margins": 19.34732162952423, + "rewards/rejected": -14.90951988697052, + "step": 710 + }, + { + "entropy": 1.271144437789917, + "epoch": 2.1303703703703705, + "grad_norm": 6.771087646484375e-05, + "learning_rate": 1.1761757443482286e-05, + "logits/chosen": 2.2690192203566064, + "logits/rejected": -1.176060724310163, + "logps/chosen": -65.25097322463989, + "logps/rejected": -325.64191341400146, + "loss": 7.844523679523263e-08, + "mean_token_accuracy": 0.6650686420500278, + "num_tokens": 6629998.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.375448957085609, + "rewards/margins": 19.631182181835175, + "rewards/rejected": -15.255733144283294, + "step": 720 + }, + { + "entropy": 1.263370680809021, + "epoch": 2.16, + "grad_norm": 0.00067901611328125, + "learning_rate": 1.1039222039359645e-05, + "logits/chosen": 2.2999157883359946, + "logits/rejected": -1.1241040776625622, + "logps/chosen": -65.3091173171997, + "logps/rejected": -327.6589548110962, + "loss": 6.326518359855982e-07, + "mean_token_accuracy": 0.6666021130979061, + "num_tokens": 6721579.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.218965843319893, + "rewards/margins": 19.43423910140991, + "rewards/rejected": -15.21527328491211, + "step": 730 + }, + { + "entropy": 1.2835220411419868, + "epoch": 2.18962962962963, + "grad_norm": 9.059906005859375e-06, + "learning_rate": 1.0333251074666609e-05, + "logits/chosen": 2.2917566691940756, + "logits/rejected": -1.2816091428578473, + "logps/chosen": -66.77954509258271, + "logps/rejected": -333.47667198181153, + "loss": 1.7912802832142916e-07, + "mean_token_accuracy": 0.6699437476694584, + "num_tokens": 6813377.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.310945153236389, + "rewards/margins": 19.620563447475433, + "rewards/rejected": -15.309618008136749, + "step": 740 + }, + { + "entropy": 1.2834552891552449, + "epoch": 2.2192592592592595, + "grad_norm": 2.8252601623535156e-05, + "learning_rate": 9.644682182758306e-06, + "logits/chosen": 2.296253967390375, + "logits/rejected": -1.1382098734020025, + "logps/chosen": -65.18716852664947, + "logps/rejected": -333.20018062591555, + "loss": 3.619171593527426e-07, + "mean_token_accuracy": 0.670500548928976, + "num_tokens": 6906265.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.34758489727974, + "rewards/margins": 19.480838096141817, + "rewards/rejected": -15.133253240585328, + "step": 750 + }, + { + "entropy": 1.2889503717422486, + "epoch": 2.2488888888888887, + "grad_norm": 3.6716461181640625e-05, + "learning_rate": 8.974332349459994e-06, + "logits/chosen": 2.274084705882701, + "logits/rejected": -1.1973438289074938, + "logps/chosen": -62.72564001083374, + "logps/rejected": -335.81445999145507, + "loss": 2.772670995909721e-07, + "mean_token_accuracy": 0.663724122941494, + "num_tokens": 7004404.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.22343917787075, + "rewards/margins": 19.607576775550843, + "rewards/rejected": -15.384137523174285, + "step": 760 + }, + { + "entropy": 1.2709729745984077, + "epoch": 2.2785185185185184, + "grad_norm": 6.818771362304688e-05, + "learning_rate": 8.322996943714672e-06, + "logits/chosen": 2.315020811648067, + "logits/rejected": -1.1844662490433007, + "logps/chosen": -60.06333501338959, + "logps/rejected": -324.07994575500487, + "loss": 6.899633149259899e-08, + "mean_token_accuracy": 0.674921452999115, + "num_tokens": 7091630.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.219157862663269, + "rewards/margins": 19.409486842155456, + "rewards/rejected": -15.190328931808471, + "step": 770 + }, + { + "entropy": 1.2802273795008658, + "epoch": 2.308148148148148, + "grad_norm": 7.677078247070312e-05, + "learning_rate": 7.691448773879257e-06, + "logits/chosen": 2.25499691013824, + "logits/rejected": -1.1512603749381776, + "logps/chosen": -63.52460696697235, + "logps/rejected": -325.2589776992798, + "loss": 1.4859768953101594e-07, + "mean_token_accuracy": 0.6620502531528473, + "num_tokens": 7182707.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.2716171652078625, + "rewards/margins": 19.37758446931839, + "rewards/rejected": -15.10596705675125, + "step": 780 + }, + { + "entropy": 1.2827171966433526, + "epoch": 2.3377777777777777, + "grad_norm": 6.645917892456055e-06, + "learning_rate": 7.080437170788723e-06, + "logits/chosen": 2.311279925102533, + "logits/rejected": -1.1746961520146812, + "logps/chosen": -59.46050326824188, + "logps/rejected": -336.5900556564331, + "loss": 3.455784508332727e-07, + "mean_token_accuracy": 0.6743544064462185, + "num_tokens": 7275656.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.199921746551991, + "rewards/margins": 19.863027906417848, + "rewards/rejected": -15.663106298446655, + "step": 790 + }, + { + "entropy": 1.2675680316984654, + "epoch": 2.3674074074074074, + "grad_norm": 0.00015926361083984375, + "learning_rate": 6.490687098676332e-06, + "logits/chosen": 2.3366430778544096, + "logits/rejected": -1.1229084532499813, + "logps/chosen": -58.34785094261169, + "logps/rejected": -331.7050970077515, + "loss": 6.806028522987617e-07, + "mean_token_accuracy": 0.6765504986047745, + "num_tokens": 7367211.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.387070021033287, + "rewards/margins": 19.79595556259155, + "rewards/rejected": -15.40888546705246, + "step": 800 + }, + { + "epoch": 2.3674074074074074, + "eval_entropy": 1.2803315270197142, + "eval_logits/chosen": 2.2436578263578553, + "eval_logits/rejected": -1.158513689404391, + "eval_logps/chosen": -68.02018330980847, + "eval_logps/rejected": -326.71273963768164, + "eval_loss": 7.024308615655173e-06, + "eval_mean_token_accuracy": 0.6631912376497175, + "eval_num_tokens": 7367211.0, + "eval_rewards/accuracies": 1.0, + "eval_rewards/chosen": 4.313107631006441, + "eval_rewards/margins": 19.479393625592852, + "eval_rewards/rejected": -15.166285921643665, + "eval_runtime": 39.4139, + "eval_samples_per_second": 7.231, + "eval_steps_per_second": 3.628, + "step": 800 + }, + { + "entropy": 1.2536483012139796, + "epoch": 2.397037037037037, + "grad_norm": 0.00010061264038085938, + "learning_rate": 5.9228982950048416e-06, + "logits/chosen": 2.306488303700283, + "logits/rejected": -1.102614074804673, + "logps/chosen": -64.7256872177124, + "logps/rejected": -316.81419982910154, + "loss": 2.7541559575183785e-07, + "mean_token_accuracy": 0.6695533588528633, + "num_tokens": 7456110.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.3090651541948315, + "rewards/margins": 19.122487926483153, + "rewards/rejected": -14.813422644138337, + "step": 810 + }, + { + "entropy": 1.2691614858806133, + "epoch": 2.4266666666666667, + "grad_norm": 3.844499588012695e-06, + "learning_rate": 5.377744440229135e-06, + "logits/chosen": 2.276143363869749, + "logits/rejected": -1.2205610514710212, + "logps/chosen": -64.77891628742218, + "logps/rejected": -330.2338497161865, + "loss": 9.86900658972445e-08, + "mean_token_accuracy": 0.6663463093340397, + "num_tokens": 7548949.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.394037923216819, + "rewards/margins": 19.882170844078065, + "rewards/rejected": -15.488132905960082, + "step": 820 + }, + { + "entropy": 1.2759164057672023, + "epoch": 2.4562962962962964, + "grad_norm": 1.5735626220703125e-05, + "learning_rate": 4.855872358475547e-06, + "logits/chosen": 2.2611198054529975, + "logits/rejected": -1.1668456534925964, + "logps/chosen": -63.698548150062564, + "logps/rejected": -323.2269037246704, + "loss": 1.5968792013154598e-07, + "mean_token_accuracy": 0.6731547065079212, + "num_tokens": 7637248.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.312624426186085, + "rewards/margins": 19.461497557163238, + "rewards/rejected": -15.148873174190522, + "step": 830 + }, + { + "entropy": 1.2611838012933732, + "epoch": 2.485925925925926, + "grad_norm": 0.0002841949462890625, + "learning_rate": 4.357901250086108e-06, + "logits/chosen": 2.2900714239429774, + "logits/rejected": -1.2470973023927523, + "logps/chosen": -62.78718073368073, + "logps/rejected": -324.9568237304687, + "loss": 1.4255396081352955e-07, + "mean_token_accuracy": 0.6746141783893108, + "num_tokens": 7726306.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.291703987121582, + "rewards/margins": 19.421786904335022, + "rewards/rejected": -15.130082988739014, + "step": 840 + }, + { + "entropy": 1.2697825357317924, + "epoch": 2.5155555555555553, + "grad_norm": 0.0002613067626953125, + "learning_rate": 3.884421956938378e-06, + "logits/chosen": 2.3862500773086817, + "logits/rejected": -1.1250058733609287, + "logps/chosen": -61.92496125698089, + "logps/rejected": -327.44327507019045, + "loss": 1.493016384301882e-07, + "mean_token_accuracy": 0.6789508298039436, + "num_tokens": 7822125.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.364183455705643, + "rewards/margins": 19.506937170028685, + "rewards/rejected": -15.142753863334656, + "step": 850 + }, + { + "entropy": 1.2504698053002357, + "epoch": 2.5451851851851854, + "grad_norm": 4.506111145019531e-05, + "learning_rate": 3.435996261412591e-06, + "logits/chosen": 2.2986260733744173, + "logits/rejected": -1.1058551351474077, + "logps/chosen": -65.89442076683045, + "logps/rejected": -330.00583362579346, + "loss": 1.4223222706277737e-07, + "mean_token_accuracy": 0.6768516503274441, + "num_tokens": 7915620.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.385465979576111, + "rewards/margins": 20.04776966571808, + "rewards/rejected": -15.662303495407105, + "step": 860 + }, + { + "entropy": 1.2795700900256635, + "epoch": 2.5748148148148147, + "grad_norm": 2.5153160095214844e-05, + "learning_rate": 3.013156219837776e-06, + "logits/chosen": 2.237134730489094, + "logits/rejected": -1.1218436377455188, + "logps/chosen": -65.73119521141052, + "logps/rejected": -322.3418972015381, + "loss": 2.536130750740995e-07, + "mean_token_accuracy": 0.665914348512888, + "num_tokens": 8006742.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.355473381280899, + "rewards/margins": 19.215320122241973, + "rewards/rejected": -14.85984662771225, + "step": 870 + }, + { + "entropy": 1.2761830180883407, + "epoch": 2.6044444444444443, + "grad_norm": 1.1742115020751953e-05, + "learning_rate": 2.6164035312078448e-06, + "logits/chosen": 2.2800606568463015, + "logits/rejected": -1.1315629358769943, + "logps/chosen": -61.99486291408539, + "logps/rejected": -326.9346130371094, + "loss": 2.3778172817401355e-07, + "mean_token_accuracy": 0.6662385709583759, + "num_tokens": 8100018.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.3306570738554, + "rewards/margins": 19.479354202747345, + "rewards/rejected": -15.148697006702424, + "step": 880 + }, + { + "entropy": 1.279340496659279, + "epoch": 2.634074074074074, + "grad_norm": 9.632110595703125e-05, + "learning_rate": 2.2462089419165777e-06, + "logits/chosen": 2.3029893365635097, + "logits/rejected": -1.171205228496424, + "logps/chosen": -65.00987663269044, + "logps/rejected": -330.74864501953124, + "loss": 6.256735787246726e-08, + "mean_token_accuracy": 0.6665534779429436, + "num_tokens": 8194303.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.474057069420814, + "rewards/margins": 19.85085200071335, + "rewards/rejected": -15.376794970035553, + "step": 890 + }, + { + "entropy": 1.2603400394320488, + "epoch": 2.6637037037037037, + "grad_norm": 2.4557113647460938e-05, + "learning_rate": 1.9030116872178316e-06, + "logits/chosen": 2.249223475648134, + "logits/rejected": -1.0936998011992798, + "logps/chosen": -64.42266466617585, + "logps/rejected": -319.57002086639403, + "loss": 1.4027624501977697e-07, + "mean_token_accuracy": 0.6666488841176033, + "num_tokens": 8285684.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.301889230310917, + "rewards/margins": 19.297891116142274, + "rewards/rejected": -14.99600180387497, + "step": 900 + }, + { + "epoch": 2.6637037037037037, + "eval_entropy": 1.2802694969243937, + "eval_logits/chosen": 2.2445470870400355, + "eval_logits/rejected": -1.159552092885801, + "eval_logps/chosen": -67.97919230694538, + "eval_logps/rejected": -326.7502327232094, + "eval_loss": 6.49735875413171e-06, + "eval_mean_token_accuracy": 0.6639414579718264, + "eval_num_tokens": 8285684.0, + "eval_rewards/accuracies": 1.0, + "eval_rewards/chosen": 4.317206760803303, + "eval_rewards/margins": 19.48724203843337, + "eval_rewards/rejected": -15.17003523553168, + "eval_runtime": 39.451, + "eval_samples_per_second": 7.224, + "eval_steps_per_second": 3.625, + "step": 900 + }, + { + "entropy": 1.2844877071678638, + "epoch": 2.6933333333333334, + "grad_norm": 2.205371856689453e-05, + "learning_rate": 1.5872189700736339e-06, + "logits/chosen": 2.341007675682582, + "logits/rejected": -1.1647508343869704, + "logps/chosen": -65.77681732177734, + "logps/rejected": -333.8214714050293, + "loss": 9.888794920698274e-08, + "mean_token_accuracy": 0.6609948001801967, + "num_tokens": 8379162.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.243448624014855, + "rewards/margins": 19.678018867969513, + "rewards/rejected": -15.434570217132569, + "step": 910 + }, + { + "entropy": 1.2820228539407252, + "epoch": 2.722962962962963, + "grad_norm": 2.9325485229492188e-05, + "learning_rate": 1.2992054780085694e-06, + "logits/chosen": 2.308397402508807, + "logits/rejected": -1.1578754052596163, + "logps/chosen": -60.28462275266647, + "logps/rejected": -329.8702238082886, + "loss": 5.674120302501251e-08, + "mean_token_accuracy": 0.6794499330222606, + "num_tokens": 8471171.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.262164607644081, + "rewards/margins": 19.57164797782898, + "rewards/rejected": -15.309483373165131, + "step": 920 + }, + { + "entropy": 1.2623218022286893, + "epoch": 2.7525925925925927, + "grad_norm": 0.00011157989501953125, + "learning_rate": 1.0393129385436824e-06, + "logits/chosen": 2.3240682024976924, + "logits/rejected": -1.0668921112986016, + "logps/chosen": -59.67337925434113, + "logps/rejected": -327.08252410888673, + "loss": 1.114461042561743e-07, + "mean_token_accuracy": 0.6755911834537983, + "num_tokens": 8562918.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.329451468586922, + "rewards/margins": 19.44830514192581, + "rewards/rejected": -15.118853759765624, + "step": 930 + }, + { + "entropy": 1.2947206407785417, + "epoch": 2.7822222222222224, + "grad_norm": 0.00011396408081054688, + "learning_rate": 8.078497137373242e-07, + "logits/chosen": 2.221305657513471, + "logits/rejected": -1.151073296194637, + "logps/chosen": -66.11703178882598, + "logps/rejected": -329.9900022506714, + "loss": 1.417975795448001e-07, + "mean_token_accuracy": 0.6663264043629169, + "num_tokens": 8655421.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.399758037924767, + "rewards/margins": 19.651811373233794, + "rewards/rejected": -15.252053284645081, + "step": 940 + }, + { + "entropy": 1.2984899178147316, + "epoch": 2.811851851851852, + "grad_norm": 0.00030517578125, + "learning_rate": 6.050904343141095e-07, + "logits/chosen": 2.22722939090554, + "logits/rejected": -1.1332568821088258, + "logps/chosen": -66.14050345420837, + "logps/rejected": -324.22646408081056, + "loss": 2.464663111823029e-07, + "mean_token_accuracy": 0.6699574515223503, + "num_tokens": 8745995.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.108328685164452, + "rewards/margins": 19.001902163028717, + "rewards/rejected": -14.893573427200318, + "step": 950 + }, + { + "entropy": 1.2638258799910544, + "epoch": 2.8414814814814813, + "grad_norm": 2.4437904357910156e-06, + "learning_rate": 4.312756738160145e-07, + "logits/chosen": 2.2900851054697773, + "logits/rejected": -1.1548115943695305, + "logps/chosen": -62.25515162944794, + "logps/rejected": -332.41545429229734, + "loss": 1.340860762866214e-07, + "mean_token_accuracy": 0.6710602834820747, + "num_tokens": 8840853.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.389033368229866, + "rewards/margins": 19.693468022346497, + "rewards/rejected": -15.304434645175935, + "step": 960 + }, + { + "entropy": 1.2863718561828137, + "epoch": 2.871111111111111, + "grad_norm": 1.8358230590820312e-05, + "learning_rate": 2.8661166316229225e-07, + "logits/chosen": 2.3169417969141524, + "logits/rejected": -1.186848696642711, + "logps/chosen": -62.72764225006104, + "logps/rejected": -326.7688617706299, + "loss": 1.440903929506021e-07, + "mean_token_accuracy": 0.6690138332545758, + "num_tokens": 8932082.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.227344951033592, + "rewards/margins": 19.289413845539094, + "rewards/rejected": -15.062068808078767, + "step": 970 + }, + { + "entropy": 1.2781345948576928, + "epoch": 2.9007407407407406, + "grad_norm": 4.887580871582031e-06, + "learning_rate": 1.7127004595681729e-07, + "logits/chosen": 2.351716086708728, + "logits/rejected": -1.1749225494133453, + "logps/chosen": -69.60287075042724, + "logps/rejected": -329.7513824462891, + "loss": 1.1806017710114248e-07, + "mean_token_accuracy": 0.6486908756196499, + "num_tokens": 9029000.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.309024539589882, + "rewards/margins": 19.48654924631119, + "rewards/rejected": -15.177524662017822, + "step": 980 + }, + { + "entropy": 1.2663434036076069, + "epoch": 2.9303703703703703, + "grad_norm": 7.063150405883789e-06, + "learning_rate": 8.538767483325383e-08, + "logits/chosen": 2.3055120496617296, + "logits/rejected": -1.0696752761558757, + "logps/chosen": -65.15313308238983, + "logps/rejected": -333.3471397399902, + "loss": 1.6038972034948528e-07, + "mean_token_accuracy": 0.6714354790747166, + "num_tokens": 9120849.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.179977935552597, + "rewards/margins": 19.668525290489196, + "rewards/rejected": -15.488547360897064, + "step": 990 + }, + { + "entropy": 1.279862828552723, + "epoch": 2.96, + "grad_norm": 3.2633543014526367e-06, + "learning_rate": 2.9066449079634406e-08, + "logits/chosen": 2.3262460395698, + "logits/rejected": -1.0832974043675088, + "logps/chosen": -65.35808026790619, + "logps/rejected": -333.4561340332031, + "loss": 1.2052357760694577e-07, + "mean_token_accuracy": 0.654293454438448, + "num_tokens": 9214314.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.2864476755261425, + "rewards/margins": 19.81372677087784, + "rewards/rejected": -15.527279019355774, + "step": 1000 + }, + { + "epoch": 2.96, + "eval_entropy": 1.280155298593161, + "eval_logits/chosen": 2.2437926653321396, + "eval_logits/rejected": -1.1593534352886283, + "eval_logps/chosen": -68.00064637777689, + "eval_logps/rejected": -326.6981409246271, + "eval_loss": 6.490902251243824e-06, + "eval_mean_token_accuracy": 0.6637652328797987, + "eval_num_tokens": 9214314.0, + "eval_rewards/accuracies": 1.0, + "eval_rewards/chosen": 4.315061373310489, + "eval_rewards/margins": 19.479887595543495, + "eval_rewards/rejected": -15.164826126365394, + "eval_runtime": 39.7099, + "eval_samples_per_second": 7.177, + "eval_steps_per_second": 3.601, + "step": 1000 + }, + { + "entropy": 1.2753226958215236, + "epoch": 2.9896296296296296, + "grad_norm": 2.09808349609375e-05, + "learning_rate": 2.3731937350224277e-09, + "logits/chosen": 2.2483844642063278, + "logits/rejected": -1.1680978999426856, + "logps/chosen": -62.2261656999588, + "logps/rejected": -334.1520866394043, + "loss": 1.8372368231212023e-07, + "mean_token_accuracy": 0.6661621876060962, + "num_tokens": 9307687.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.231814455986023, + "rewards/margins": 19.692088031768797, + "rewards/rejected": -15.460273444652557, + "step": 1010 + }, + { + "epoch": 2.9985185185185186, + "eval_entropy": 1.280155298593161, + "eval_logits/chosen": 2.2437926653321396, + "eval_logits/rejected": -1.1593534352886283, + "eval_logps/chosen": -68.00064637777689, + "eval_logps/rejected": -326.6981409246271, + "eval_loss": 6.490902251243824e-06, + "eval_mean_token_accuracy": 0.6637652328797987, + "eval_num_tokens": 9336284.0, + "eval_rewards/accuracies": 1.0, + "eval_rewards/chosen": 4.315061373310489, + "eval_rewards/margins": 19.479887595543495, + "eval_rewards/rejected": -15.164826126365394, + "eval_runtime": 39.6872, + "eval_samples_per_second": 7.181, + "eval_steps_per_second": 3.603, + "step": 1013 + } + ], + "logging_steps": 10, + "max_steps": 1013, + "num_input_tokens_seen": 0, + "num_train_epochs": 3, + "save_steps": 100, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": true + }, + "attributes": {} + } + }, + "total_flos": 4.939535659708662e+17, + "train_batch_size": 2, + "trial_name": null, + "trial_params": null +} diff --git a/checkpoint-1013/training_args.bin b/checkpoint-1013/training_args.bin new file mode 100644 index 0000000..5ac503c --- /dev/null +++ b/checkpoint-1013/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9511438415c2456684b2907b1d3a2888ce92542e75957d8ee17f0b9dda11c01a +size 7249 diff --git a/checkpoint-900/chat_template.jinja b/checkpoint-900/chat_template.jinja new file mode 100644 index 0000000..28028c0 --- /dev/null +++ b/checkpoint-900/chat_template.jinja @@ -0,0 +1,54 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0]['role'] == 'system' %} + {{- messages[0]['content'] }} + {%- else %} + {{- 'You are a helpful assistant.' }} + {%- endif %} + {{- "\n\n# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0]['role'] == 'system' %} + {{- '<|im_start|>system\n' + messages[0]['content'] + '<|im_end|>\n' }} + {%- else %} + {{- '<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- for message in messages %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) or (message.role == "assistant" and not message.tool_calls) %} + {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {{- '<|im_start|>' + message.role }} + {%- if message.content %} + {{- '\n' + message.content }} + {%- endif %} + {%- for tool_call in message.tool_calls %} + {%- if tool_call.function is defined %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {{- tool_call.arguments | tojson }} + {{- '}\n' }} + {%- endfor %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- message.content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} +{%- endif %} diff --git a/checkpoint-900/config.json b/checkpoint-900/config.json new file mode 100644 index 0000000..6b475aa --- /dev/null +++ b/checkpoint-900/config.json @@ -0,0 +1,62 @@ +{ + "architectures": [ + "Qwen2ForCausalLM" + ], + "attention_dropout": 0.0, + "bos_token_id": null, + "dtype": "bfloat16", + "eos_token_id": 151643, + "hidden_act": "silu", + "hidden_size": 3584, + "initializer_range": 0.02, + "intermediate_size": 18944, + "layer_types": [ + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention" + ], + "max_position_embeddings": 131072, + "max_window_layers": 28, + "model_type": "qwen2", + "num_attention_heads": 28, + "num_hidden_layers": 28, + "num_key_value_heads": 4, + "pad_token_id": 151643, + "rms_norm_eps": 1e-06, + "rope_parameters": { + "rope_theta": 1000000.0, + "rope_type": "default" + }, + "sliding_window": null, + "tie_word_embeddings": false, + "transformers_version": "5.5.4", + "use_cache": false, + "use_mrope": false, + "use_sliding_window": false, + "vocab_size": 152064 +} diff --git a/checkpoint-900/generation_config.json b/checkpoint-900/generation_config.json new file mode 100644 index 0000000..0fb69a0 --- /dev/null +++ b/checkpoint-900/generation_config.json @@ -0,0 +1,9 @@ +{ + "do_sample": true, + "eos_token_id": [ + 151643 + ], + "max_new_tokens": 2048, + "pad_token_id": 151643, + "transformers_version": "5.5.4" +} diff --git a/checkpoint-900/model.safetensors b/checkpoint-900/model.safetensors new file mode 100644 index 0000000..a2403b7 --- /dev/null +++ b/checkpoint-900/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7ad10f66da5f83217b44c52f55f8df99f7e8c8e5ff5b69f677cf8dda28d3f5d2 +size 15231272152 diff --git a/checkpoint-900/optimizer.pt b/checkpoint-900/optimizer.pt new file mode 100644 index 0000000..5ec9b10 --- /dev/null +++ b/checkpoint-900/optimizer.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2e9e28b5395219ce99a934cbe994e0b9641937c66c255eb69a3db45c53949328 +size 18724493813 diff --git a/checkpoint-900/rng_state.pth b/checkpoint-900/rng_state.pth new file mode 100644 index 0000000..fbfc9df --- /dev/null +++ b/checkpoint-900/rng_state.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:236304ae89e49aae8260113165ee63419b9b745f79120014328a7fa31ed79b42 +size 14645 diff --git a/checkpoint-900/scheduler.pt b/checkpoint-900/scheduler.pt new file mode 100644 index 0000000..2cb7084 --- /dev/null +++ b/checkpoint-900/scheduler.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dfd0d74b5a4cf4dd0431fbe46896e959595a03be7897d929a63329d309c1c2ef +size 1465 diff --git a/checkpoint-900/tokenizer.json b/checkpoint-900/tokenizer.json new file mode 100644 index 0000000..34510ff --- /dev/null +++ b/checkpoint-900/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3fd169731d2cbde95e10bf356d66d5997fd885dd8dbb6fb4684da3f23b2585d8 +size 11421892 diff --git a/checkpoint-900/tokenizer_config.json b/checkpoint-900/tokenizer_config.json new file mode 100644 index 0000000..c960ecf --- /dev/null +++ b/checkpoint-900/tokenizer_config.json @@ -0,0 +1,29 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/checkpoint-900/trainer_state.json b/checkpoint-900/trainer_state.json new file mode 100644 index 0000000..5f3b263 --- /dev/null +++ b/checkpoint-900/trainer_state.json @@ -0,0 +1,1862 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 2.6637037037037037, + "eval_steps": 100, + "global_step": 900, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0, + "eval_entropy": 1.7289148477407603, + "eval_logits/chosen": 0.8636333912138532, + "eval_logits/rejected": -0.6579137279251289, + "eval_logps/chosen": -111.15125904216633, + "eval_logps/rejected": -175.04988199514108, + "eval_loss": 0.6931471228599548, + "eval_mean_token_accuracy": 0.4934375546612106, + "eval_num_tokens": 0.0, + "eval_rewards/accuracies": 0.0, + "eval_rewards/chosen": 0.0, + "eval_rewards/margins": 0.0, + "eval_rewards/rejected": 0.0, + "eval_runtime": 39.0805, + "eval_samples_per_second": 7.293, + "eval_steps_per_second": 3.659, + "step": 0 + }, + { + "entropy": 1.7504533976316452, + "epoch": 0.002962962962962963, + "grad_norm": 83.5, + "learning_rate": 0.0, + "logits/chosen": 1.08207520218349, + "logits/rejected": -0.5940652974601717, + "logps/chosen": -110.63330745697021, + "logps/rejected": -182.49396514892578, + "loss": 0.6931471824645996, + "mean_token_accuracy": 0.5048413500189781, + "num_tokens": 10169.0, + "rewards/accuracies": 0.0, + "rewards/chosen": 0.0, + "rewards/margins": 0.0, + "rewards/rejected": 0.0, + "step": 1 + }, + { + "entropy": 1.7299097461832895, + "epoch": 0.02962962962962963, + "grad_norm": 29.875, + "learning_rate": 4.455445544554456e-06, + "logits/chosen": 0.8212555533910425, + "logits/rejected": -0.6693587809238846, + "logps/chosen": -100.81309244367812, + "logps/rejected": -171.8307899898953, + "loss": 0.5316412713792589, + "mean_token_accuracy": 0.5100594382319186, + "num_tokens": 89122.0, + "rewards/accuracies": 0.8125, + "rewards/chosen": 0.306247040655257, + "rewards/margins": 0.41913801974280634, + "rewards/rejected": -0.11289098787690616, + "step": 10 + }, + { + "entropy": 1.5854810774326324, + "epoch": 0.05925925925925926, + "grad_norm": 0.07470703125, + "learning_rate": 9.405940594059407e-06, + "logits/chosen": 1.2175614897456257, + "logits/rejected": -0.8402934758508309, + "logps/chosen": -77.16593537330627, + "logps/rejected": -196.31047840118407, + "loss": 0.036623942852020266, + "mean_token_accuracy": 0.5996901635080576, + "num_tokens": 182773.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 2.910937874764204, + "rewards/margins": 5.368858252465725, + "rewards/rejected": -2.4579203579574824, + "step": 20 + }, + { + "entropy": 1.4120451480150222, + "epoch": 0.08888888888888889, + "grad_norm": 0.019287109375, + "learning_rate": 1.4356435643564355e-05, + "logits/chosen": 2.264547569453187, + "logits/rejected": -1.0346384454567032, + "logps/chosen": -64.55358953475952, + "logps/rejected": -259.64974784851074, + "loss": 6.736992509104311e-05, + "mean_token_accuracy": 0.6582182548940182, + "num_tokens": 274816.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.462539532780648, + "rewards/margins": 12.943859684467316, + "rewards/rejected": -8.481320106983185, + "step": 30 + }, + { + "entropy": 1.355741088092327, + "epoch": 0.11851851851851852, + "grad_norm": 0.0004863739013671875, + "learning_rate": 1.930693069306931e-05, + "logits/chosen": 2.3368093516258224, + "logits/rejected": -1.142538018090554, + "logps/chosen": -61.829122233390805, + "logps/rejected": -295.9105268478394, + "loss": 3.1130424758885055e-06, + "mean_token_accuracy": 0.6669795714318753, + "num_tokens": 367268.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.366855585575104, + "rewards/margins": 16.085866248607637, + "rewards/rejected": -11.719010663032531, + "step": 40 + }, + { + "entropy": 1.320019119977951, + "epoch": 0.14814814814814814, + "grad_norm": 0.0001201629638671875, + "learning_rate": 2.4257425742574257e-05, + "logits/chosen": 2.315207261423255, + "logits/rejected": -1.0571934877438707, + "logps/chosen": -63.68361803293228, + "logps/rejected": -302.812979888916, + "loss": 7.578286749776453e-06, + "mean_token_accuracy": 0.6602454461157322, + "num_tokens": 461524.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.11230229139328, + "rewards/margins": 16.546387827396394, + "rewards/rejected": -12.43408569097519, + "step": 50 + }, + { + "entropy": 1.326953475177288, + "epoch": 0.17777777777777778, + "grad_norm": 0.0003871917724609375, + "learning_rate": 2.9207920792079208e-05, + "logits/chosen": 2.2619415245350254, + "logits/rejected": -1.0863394894289136, + "logps/chosen": -63.424767637252806, + "logps/rejected": -304.6159143447876, + "loss": 2.0567724277498202e-06, + "mean_token_accuracy": 0.6664863616228104, + "num_tokens": 555169.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.521066862344742, + "rewards/margins": 17.257024788856505, + "rewards/rejected": -12.735957789421082, + "step": 60 + }, + { + "entropy": 1.3109250903129577, + "epoch": 0.2074074074074074, + "grad_norm": 1.0609626770019531e-05, + "learning_rate": 3.415841584158416e-05, + "logits/chosen": 2.3168210798300413, + "logits/rejected": -1.1463735020520454, + "logps/chosen": -62.93964213132858, + "logps/rejected": -301.49362087249756, + "loss": 2.6633450761437417e-05, + "mean_token_accuracy": 0.6684455543756485, + "num_tokens": 646219.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.369684019684792, + "rewards/margins": 17.029080402851104, + "rewards/rejected": -12.659396553039551, + "step": 70 + }, + { + "entropy": 1.3183406174182892, + "epoch": 0.23703703703703705, + "grad_norm": 8.463859558105469e-06, + "learning_rate": 3.910891089108911e-05, + "logits/chosen": 2.2528845473791295, + "logits/rejected": -1.1451393864081993, + "logps/chosen": -64.70127091407775, + "logps/rejected": -313.31810626983645, + "loss": 2.219452881035977e-07, + "mean_token_accuracy": 0.6682900048792362, + "num_tokens": 741395.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.620437514781952, + "rewards/margins": 18.064030134677886, + "rewards/rejected": -13.443592536449433, + "step": 80 + }, + { + "entropy": 1.306735871732235, + "epoch": 0.26666666666666666, + "grad_norm": 0.00014209747314453125, + "learning_rate": 4.405940594059406e-05, + "logits/chosen": 2.2645358825214656, + "logits/rejected": -1.1447062023763102, + "logps/chosen": -62.47025227546692, + "logps/rejected": -309.65871505737306, + "loss": 6.8541798100341115e-06, + "mean_token_accuracy": 0.6588799960911274, + "num_tokens": 833512.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.536789815127849, + "rewards/margins": 17.808473789691924, + "rewards/rejected": -13.271684014797211, + "step": 90 + }, + { + "entropy": 1.3041012182831764, + "epoch": 0.2962962962962963, + "grad_norm": 0.0001277923583984375, + "learning_rate": 4.9009900990099014e-05, + "logits/chosen": 2.3280876455235577, + "logits/rejected": -1.0593703708195146, + "logps/chosen": -57.693548345565794, + "logps/rejected": -307.1459705352783, + "loss": 2.0000723452540113e-06, + "mean_token_accuracy": 0.6723956800997257, + "num_tokens": 925332.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.472308248281479, + "rewards/margins": 17.747703313827515, + "rewards/rejected": -13.275395059585572, + "step": 100 + }, + { + "epoch": 0.2962962962962963, + "eval_entropy": 1.3091890520149179, + "eval_logits/chosen": 2.2560516885971, + "eval_logits/rejected": -1.1380674279781138, + "eval_logps/chosen": -66.05370368157233, + "eval_logps/rejected": -310.2874222335282, + "eval_loss": 3.520939208101481e-05, + "eval_mean_token_accuracy": 0.662857598358101, + "eval_num_tokens": 925332.0, + "eval_rewards/accuracies": 1.0, + "eval_rewards/chosen": 4.5097555984983915, + "eval_rewards/margins": 18.033509881346376, + "eval_rewards/rejected": -13.523754186563558, + "eval_runtime": 38.3237, + "eval_samples_per_second": 7.437, + "eval_steps_per_second": 3.731, + "step": 100 + }, + { + "entropy": 1.284974904358387, + "epoch": 0.32592592592592595, + "grad_norm": 0.000812530517578125, + "learning_rate": 4.999050767562379e-05, + "logits/chosen": 2.3396127391184693, + "logits/rejected": -1.1138358422485608, + "logps/chosen": -61.88268647193909, + "logps/rejected": -304.68117294311526, + "loss": 3.422158624744043e-06, + "mean_token_accuracy": 0.6786789156496524, + "num_tokens": 1016416.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.462823995947838, + "rewards/margins": 17.767807400226594, + "rewards/rejected": -13.304983520507813, + "step": 110 + }, + { + "entropy": 1.2844461277127266, + "epoch": 0.35555555555555557, + "grad_norm": 0.00010204315185546875, + "learning_rate": 4.9951957461646706e-05, + "logits/chosen": 2.3448500846553584, + "logits/rejected": -1.0939366953296292, + "logps/chosen": -61.62471632957458, + "logps/rejected": -307.099747467041, + "loss": 1.316130146733485e-06, + "mean_token_accuracy": 0.6710287682712078, + "num_tokens": 1105772.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.635418501496315, + "rewards/margins": 18.310668289661407, + "rewards/rejected": -13.675249755382538, + "step": 120 + }, + { + "entropy": 1.2960298083722592, + "epoch": 0.3851851851851852, + "grad_norm": 0.0009765625, + "learning_rate": 4.988380179235843e-05, + "logits/chosen": 2.329744072704608, + "logits/rejected": -1.0829270884851314, + "logps/chosen": -60.90654925107956, + "logps/rejected": -320.0984771728516, + "loss": 2.281175875396002e-06, + "mean_token_accuracy": 0.6639963097870349, + "num_tokens": 1198464.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.473316392302513, + "rewards/margins": 18.291567063331605, + "rewards/rejected": -13.818250668048858, + "step": 130 + }, + { + "entropy": 1.2743454918265342, + "epoch": 0.4148148148148148, + "grad_norm": 0.00115966796875, + "learning_rate": 4.9786121534345265e-05, + "logits/chosen": 2.310538935492521, + "logits/rejected": -1.1208351833476473, + "logps/chosen": -58.558825969696045, + "logps/rejected": -314.1108278274536, + "loss": 1.2155161130067428e-06, + "mean_token_accuracy": 0.674963940680027, + "num_tokens": 1289352.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.255219912528991, + "rewards/margins": 18.211954617500304, + "rewards/rejected": -13.956734585762025, + "step": 140 + }, + { + "entropy": 1.2854739636182786, + "epoch": 0.4444444444444444, + "grad_norm": 0.0001659393310546875, + "learning_rate": 4.965903258506806e-05, + "logits/chosen": 2.33276765322688, + "logits/rejected": -1.1813850704996935, + "logps/chosen": -60.029894495010375, + "logps/rejected": -315.8014057159424, + "loss": 4.05059563490795e-07, + "mean_token_accuracy": 0.6699102215468884, + "num_tokens": 1380079.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.356227645277977, + "rewards/margins": 18.526466822624208, + "rewards/rejected": -14.170239269733429, + "step": 150 + }, + { + "entropy": 1.3013202093541623, + "epoch": 0.4740740740740741, + "grad_norm": 1.0132789611816406e-05, + "learning_rate": 4.950268573535012e-05, + "logits/chosen": 2.3245691177602428, + "logits/rejected": -1.0899626289102018, + "logps/chosen": -62.87379167079926, + "logps/rejected": -325.68464183807373, + "loss": 2.103474116665893e-07, + "mean_token_accuracy": 0.6679917454719544, + "num_tokens": 1474262.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.370475701987743, + "rewards/margins": 18.631532168388368, + "rewards/rejected": -14.261056447029114, + "step": 160 + }, + { + "entropy": 1.2932921536266804, + "epoch": 0.5037037037037037, + "grad_norm": 0.00017070770263671875, + "learning_rate": 4.93172664904641e-05, + "logits/chosen": 2.199208436926322, + "logits/rejected": -1.1313941206494675, + "logps/chosen": -66.81836426258087, + "logps/rejected": -313.0147771835327, + "loss": 5.84564577366109e-07, + "mean_token_accuracy": 0.6591029770672321, + "num_tokens": 1567351.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.2477530300617214, + "rewards/margins": 18.021551287174226, + "rewards/rejected": -13.773798143863678, + "step": 170 + }, + { + "entropy": 1.2906752578914165, + "epoch": 0.5333333333333333, + "grad_norm": 1.1622905731201172e-05, + "learning_rate": 4.9102994850030344e-05, + "logits/chosen": 2.257452431422654, + "logits/rejected": -1.1859710789627558, + "logps/chosen": -60.773719620704654, + "logps/rejected": -319.68780155181884, + "loss": 4.461732260097051e-07, + "mean_token_accuracy": 0.6724127769470215, + "num_tokens": 1659070.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.278084766864777, + "rewards/margins": 18.284436917304994, + "rewards/rejected": -14.006352269649506, + "step": 180 + }, + { + "entropy": 1.3018329843878746, + "epoch": 0.562962962962963, + "grad_norm": 6.4849853515625e-05, + "learning_rate": 4.886012504698769e-05, + "logits/chosen": 2.301720358967305, + "logits/rejected": -1.0489440936301595, + "logps/chosen": -61.81112604141235, + "logps/rejected": -315.3358419418335, + "loss": 5.486400823428994e-07, + "mean_token_accuracy": 0.669282128661871, + "num_tokens": 1750181.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.455310028791428, + "rewards/margins": 18.440570056438446, + "rewards/rejected": -13.985260021686553, + "step": 190 + }, + { + "entropy": 1.3009084612131119, + "epoch": 0.5925925925925926, + "grad_norm": 0.0087890625, + "learning_rate": 4.858894524594653e-05, + "logits/chosen": 2.2154390003090785, + "logits/rejected": -1.1232223850457532, + "logps/chosen": -66.61707370281219, + "logps/rejected": -316.3582260131836, + "loss": 5.881736069568433e-06, + "mean_token_accuracy": 0.6558836951851845, + "num_tokens": 1841453.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.423358170688152, + "rewards/margins": 18.573253047466277, + "rewards/rejected": -14.149895036220551, + "step": 200 + }, + { + "epoch": 0.5925925925925926, + "eval_entropy": 1.294411522108358, + "eval_logits/chosen": 2.25081917507982, + "eval_logits/rejected": -1.1530030385301466, + "eval_logps/chosen": -66.83867399842589, + "eval_logps/rejected": -318.13998082301, + "eval_loss": 1.2471568879846018e-05, + "eval_mean_token_accuracy": 0.6633061449844521, + "eval_num_tokens": 1841453.0, + "eval_rewards/accuracies": 1.0, + "eval_rewards/chosen": 4.431258590904982, + "eval_rewards/margins": 18.740268787304004, + "eval_rewards/rejected": -14.309010145547507, + "eval_runtime": 38.3878, + "eval_samples_per_second": 7.424, + "eval_steps_per_second": 3.725, + "step": 200 + }, + { + "entropy": 1.2944884866476059, + "epoch": 0.6222222222222222, + "grad_norm": 5.125999450683594e-05, + "learning_rate": 4.8289777201281974e-05, + "logits/chosen": 2.3454347980352996, + "logits/rejected": -1.1855823049416294, + "logps/chosen": -64.12132394313812, + "logps/rejected": -325.72925930023194, + "loss": 2.409943363090861e-07, + "mean_token_accuracy": 0.6670842688530684, + "num_tokens": 1935659.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.459967152774334, + "rewards/margins": 19.128938269615173, + "rewards/rejected": -14.668970954418182, + "step": 210 + }, + { + "entropy": 1.2553015358746051, + "epoch": 0.6518518518518519, + "grad_norm": 4.76837158203125e-05, + "learning_rate": 4.796297587537286e-05, + "logits/chosen": 2.308509979398422, + "logits/rejected": -1.0750038709449357, + "logps/chosen": -58.983665919303895, + "logps/rejected": -315.6023241043091, + "loss": 3.415452965782606e-07, + "mean_token_accuracy": 0.6794103197753429, + "num_tokens": 2029268.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.352584506571293, + "rewards/margins": 18.749567639827728, + "rewards/rejected": -14.396983158588409, + "step": 220 + }, + { + "entropy": 1.2749639593064785, + "epoch": 0.6814814814814815, + "grad_norm": 2.7894973754882812e-05, + "learning_rate": 4.760892901743944e-05, + "logits/chosen": 2.263447201357434, + "logits/rejected": -1.0746111454871972, + "logps/chosen": -64.98015263080597, + "logps/rejected": -320.59440212249757, + "loss": 1.3830200259690174e-06, + "mean_token_accuracy": 0.6660085044801235, + "num_tokens": 2123260.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.317273432016373, + "rewards/margins": 18.91294218301773, + "rewards/rejected": -14.59566867351532, + "step": 230 + }, + { + "entropy": 1.2890466079115868, + "epoch": 0.7111111111111111, + "grad_norm": 2.7894973754882812e-05, + "learning_rate": 4.722805670347963e-05, + "logits/chosen": 2.3375335604609835, + "logits/rejected": -1.1522054103145247, + "logps/chosen": -62.949660015106204, + "logps/rejected": -325.21918926239016, + "loss": 1.8574786508906982e-07, + "mean_token_accuracy": 0.6677135609090328, + "num_tokens": 2211888.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.302155223488808, + "rewards/margins": 19.20953975915909, + "rewards/rejected": -14.90738466978073, + "step": 240 + }, + { + "entropy": 1.297672052681446, + "epoch": 0.7407407407407407, + "grad_norm": 1.3649463653564453e-05, + "learning_rate": 4.6820810837849536e-05, + "logits/chosen": 2.3567186399215143, + "logits/rejected": -1.2153430976155177, + "logps/chosen": -61.082769107818606, + "logps/rejected": -327.5022741317749, + "loss": 4.804733634955483e-07, + "mean_token_accuracy": 0.6871423445641994, + "num_tokens": 2307633.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.415084969997406, + "rewards/margins": 19.260318982601166, + "rewards/rejected": -14.845234000682831, + "step": 250 + }, + { + "entropy": 1.289575246721506, + "epoch": 0.7703703703703704, + "grad_norm": 0.00013637542724609375, + "learning_rate": 4.63876746170797e-05, + "logits/chosen": 2.2677862381437905, + "logits/rejected": -1.1125468158062497, + "logps/chosen": -67.2604984998703, + "logps/rejected": -321.5008083343506, + "loss": 1.5667228581150994e-07, + "mean_token_accuracy": 0.6551826104521752, + "num_tokens": 2404339.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.320639500021935, + "rewards/margins": 18.89977525472641, + "rewards/rejected": -14.57913579940796, + "step": 260 + }, + { + "entropy": 1.2907186925411225, + "epoch": 0.8, + "grad_norm": 2.5987625122070312e-05, + "learning_rate": 4.592916195656322e-05, + "logits/chosen": 2.337716368388003, + "logits/rejected": -1.1807755553775976, + "logps/chosen": -61.246209836006166, + "logps/rejected": -321.838525390625, + "loss": 3.219545988031314e-07, + "mean_token_accuracy": 0.67101164534688, + "num_tokens": 2493582.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.55239104628563, + "rewards/margins": 19.164004135131837, + "rewards/rejected": -14.611613166332244, + "step": 270 + }, + { + "entropy": 1.2973388336598872, + "epoch": 0.8296296296296296, + "grad_norm": 1.0728836059570312e-05, + "learning_rate": 4.544581688079602e-05, + "logits/chosen": 2.2622078397191254, + "logits/rejected": -1.1987081179562755, + "logps/chosen": -62.545769691467285, + "logps/rejected": -328.43379402160645, + "loss": 1.6840259604578022e-07, + "mean_token_accuracy": 0.6727346412837505, + "num_tokens": 2583501.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.605609413981438, + "rewards/margins": 19.350289702415466, + "rewards/rejected": -14.744680321216583, + "step": 280 + }, + { + "entropy": 1.270925548672676, + "epoch": 0.8592592592592593, + "grad_norm": 1.4722347259521484e-05, + "learning_rate": 4.4938212877892725e-05, + "logits/chosen": 2.304147609404456, + "logits/rejected": -1.1603299402866345, + "logps/chosen": -64.19616587162018, + "logps/rejected": -321.61998023986814, + "loss": 1.092423531190434e-07, + "mean_token_accuracy": 0.6694903835654259, + "num_tokens": 2673092.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.521212270855903, + "rewards/margins": 19.481856608390807, + "rewards/rejected": -14.960644149780274, + "step": 290 + }, + { + "entropy": 1.2587753705680371, + "epoch": 0.8888888888888888, + "grad_norm": 8.046627044677734e-06, + "learning_rate": 4.4406952219143936e-05, + "logits/chosen": 2.2447284227823454, + "logits/rejected": -1.2047798430182726, + "logps/chosen": -63.220883178710935, + "logps/rejected": -316.7784074783325, + "loss": 9.590096851752606e-07, + "mean_token_accuracy": 0.6734147310256958, + "num_tokens": 2765264.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.453023025393486, + "rewards/margins": 19.15658880472183, + "rewards/rejected": -14.703565871715545, + "step": 300 + }, + { + "epoch": 0.8888888888888888, + "eval_entropy": 1.2864730945833913, + "eval_logits/chosen": 2.245142647914562, + "eval_logits/rejected": -1.1577701959581186, + "eval_logps/chosen": -67.51500383123651, + "eval_logps/rejected": -323.1507350681545, + "eval_loss": 9.1151678134338e-06, + "eval_mean_token_accuracy": 0.6627988360978506, + "eval_num_tokens": 2765264.0, + "eval_rewards/accuracies": 1.0, + "eval_rewards/chosen": 4.3636256072904684, + "eval_rewards/margins": 19.173711116497334, + "eval_rewards/rejected": -14.810085516709547, + "eval_runtime": 38.408, + "eval_samples_per_second": 7.42, + "eval_steps_per_second": 3.723, + "step": 300 + }, + { + "entropy": 1.2768845409154892, + "epoch": 0.9185185185185185, + "grad_norm": 4.315376281738281e-05, + "learning_rate": 4.385266524442241e-05, + "logits/chosen": 2.2955878654382667, + "logits/rejected": -1.200506409819956, + "logps/chosen": -62.47047922611237, + "logps/rejected": -329.1678291320801, + "loss": 2.0756017420353602e-07, + "mean_token_accuracy": 0.6736313477158546, + "num_tokens": 2860290.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.335641312599182, + "rewards/margins": 19.35282862186432, + "rewards/rejected": -15.01718727350235, + "step": 310 + }, + { + "entropy": 1.267455793917179, + "epoch": 0.9481481481481482, + "grad_norm": 2.2649765014648438e-05, + "learning_rate": 4.3276009614285825e-05, + "logits/chosen": 2.301217628396178, + "logits/rejected": -1.1612455940414776, + "logps/chosen": -60.544256925582886, + "logps/rejected": -325.25582218170166, + "loss": 8.637363748675853e-08, + "mean_token_accuracy": 0.6695008479058743, + "num_tokens": 2951326.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.26083664894104, + "rewards/margins": 19.293893361091612, + "rewards/rejected": -15.033056676387787, + "step": 320 + }, + { + "entropy": 1.2802438855171203, + "epoch": 0.9777777777777777, + "grad_norm": 3.1441450119018555e-06, + "learning_rate": 4.267766952966369e-05, + "logits/chosen": 2.3043263280692856, + "logits/rejected": -1.1247859920954724, + "logps/chosen": -65.50963315963745, + "logps/rejected": -327.67705821990967, + "loss": 1.6503474398632533e-06, + "mean_token_accuracy": 0.6599100351333618, + "num_tokens": 3045520.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.30358607918024, + "rewards/margins": 19.168729376792907, + "rewards/rejected": -14.865143322944641, + "step": 330 + }, + { + "entropy": 1.2861412669482983, + "epoch": 1.005925925925926, + "grad_norm": 5.793571472167969e-05, + "learning_rate": 4.205835492005405e-05, + "logits/chosen": 2.319402775016898, + "logits/rejected": -1.2147055357233236, + "logps/chosen": -67.06965022338063, + "logps/rejected": -330.37220262226305, + "loss": 3.5803138871415284e-07, + "mean_token_accuracy": 0.6624507841311003, + "num_tokens": 3130709.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.281106482603048, + "rewards/margins": 19.5897334500363, + "rewards/rejected": -15.308626877634149, + "step": 340 + }, + { + "entropy": 1.2823633886873722, + "epoch": 1.0355555555555556, + "grad_norm": 6.008148193359375e-05, + "learning_rate": 4.141880060119336e-05, + "logits/chosen": 2.339569692570856, + "logits/rejected": -1.147382590046395, + "logps/chosen": -60.31965174674988, + "logps/rejected": -335.1733335494995, + "loss": 1.532446731289383e-07, + "mean_token_accuracy": 0.6735855638980865, + "num_tokens": 3225369.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.492888362705708, + "rewards/margins": 19.827855718135833, + "rewards/rejected": -15.334967303276063, + "step": 350 + }, + { + "entropy": 1.2730201914906503, + "epoch": 1.0651851851851852, + "grad_norm": 0.0001926422119140625, + "learning_rate": 4.075976540319888e-05, + "logits/chosen": 2.3003171439882366, + "logits/rejected": -1.2015335101578284, + "logps/chosen": -60.594508814811704, + "logps/rejected": -329.16415615081786, + "loss": 4.122233804082498e-07, + "mean_token_accuracy": 0.6743721053004265, + "num_tokens": 3318818.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.401582342386246, + "rewards/margins": 19.628432750701904, + "rewards/rejected": -15.22685045003891, + "step": 360 + }, + { + "entropy": 1.2773501306772232, + "epoch": 1.094814814814815, + "grad_norm": 0.000591278076171875, + "learning_rate": 4.0082031270217976e-05, + "logits/chosen": 2.2876778332841776, + "logits/rejected": -1.154664545777339, + "logps/chosen": -63.76930930614471, + "logps/rejected": -319.00229358673096, + "loss": 1.1508095667522866e-06, + "mean_token_accuracy": 0.6718519743531942, + "num_tokens": 3410676.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.197161988914013, + "rewards/margins": 19.0069660782814, + "rewards/rejected": -14.809803891181947, + "step": 370 + }, + { + "entropy": 1.2946350619196891, + "epoch": 1.1244444444444444, + "grad_norm": 0.00022411346435546875, + "learning_rate": 3.9386402332652754e-05, + "logits/chosen": 2.293869972488772, + "logits/rejected": -1.1689247316110847, + "logps/chosen": -64.16428699493409, + "logps/rejected": -339.3799625396729, + "loss": 1.2371945103950566e-07, + "mean_token_accuracy": 0.664111914485693, + "num_tokens": 3504356.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.358540344238281, + "rewards/margins": 19.867380928993224, + "rewards/rejected": -15.508840537071228, + "step": 380 + }, + { + "entropy": 1.2926758706569672, + "epoch": 1.154074074074074, + "grad_norm": 3.6954879760742188e-06, + "learning_rate": 3.867370395306068e-05, + "logits/chosen": 2.1766569425767703, + "logits/rejected": -1.087643152794101, + "logps/chosen": -69.47466747760772, + "logps/rejected": -327.71664810180664, + "loss": 1.194763967760082e-07, + "mean_token_accuracy": 0.6569934625178575, + "num_tokens": 3594396.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.461885306239128, + "rewards/margins": 19.697250258922576, + "rewards/rejected": -15.235364770889282, + "step": 390 + }, + { + "entropy": 1.2546571284532546, + "epoch": 1.1837037037037037, + "grad_norm": 8.940696716308594e-06, + "learning_rate": 3.794478174686328e-05, + "logits/chosen": 2.28344883152218, + "logits/rejected": -1.1421346511748105, + "logps/chosen": -64.66728138923645, + "logps/rejected": -324.4414520263672, + "loss": 7.843665343898465e-08, + "mean_token_accuracy": 0.662892397120595, + "num_tokens": 3682850.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.419300290942192, + "rewards/margins": 19.782128620147706, + "rewards/rejected": -15.362828314304352, + "step": 400 + }, + { + "epoch": 1.1837037037037037, + "eval_entropy": 1.2814960267160322, + "eval_logits/chosen": 2.2466129676027395, + "eval_logits/rejected": -1.158793695801587, + "eval_logps/chosen": -67.91167767398007, + "eval_logps/rejected": -326.0148432804988, + "eval_loss": 7.35781259209034e-06, + "eval_mean_token_accuracy": 0.6629326431067674, + "eval_num_tokens": 3682850.0, + "eval_rewards/accuracies": 1.0, + "eval_rewards/chosen": 4.323958210178189, + "eval_rewards/margins": 19.420454478764032, + "eval_rewards/rejected": -15.096496268585845, + "eval_runtime": 38.5811, + "eval_samples_per_second": 7.387, + "eval_steps_per_second": 3.706, + "step": 400 + }, + { + "entropy": 1.2794287726283073, + "epoch": 1.2133333333333334, + "grad_norm": 6.0498714447021484e-06, + "learning_rate": 3.720050057902496e-05, + "logits/chosen": 2.2348389102512294, + "logits/rejected": -1.20443931083074, + "logps/chosen": -61.86296122074127, + "logps/rejected": -331.40450420379636, + "loss": 1.5817265648365718e-07, + "mean_token_accuracy": 0.6723401784896851, + "num_tokens": 3776120.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.320593364536762, + "rewards/margins": 19.594025874137877, + "rewards/rejected": -15.273432505130767, + "step": 410 + }, + { + "entropy": 1.2816326960921287, + "epoch": 1.242962962962963, + "grad_norm": 2.7865171432495117e-06, + "learning_rate": 3.644174353789204e-05, + "logits/chosen": 2.2707540738720113, + "logits/rejected": -1.191927789160487, + "logps/chosen": -64.48058602809905, + "logps/rejected": -332.1292999267578, + "loss": 9.944698149411124e-08, + "mean_token_accuracy": 0.6633661832660437, + "num_tokens": 3868574.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.300156235694885, + "rewards/margins": 19.600781667232514, + "rewards/rejected": -15.30062551498413, + "step": 420 + }, + { + "entropy": 1.2759614169597626, + "epoch": 1.2725925925925927, + "grad_norm": 6.29425048828125e-05, + "learning_rate": 3.5669410887410095e-05, + "logits/chosen": 2.3054515861422304, + "logits/rejected": -1.1359497002203924, + "logps/chosen": -64.89167664051055, + "logps/rejected": -334.02518730163575, + "loss": 2.404902943453635e-07, + "mean_token_accuracy": 0.6750707887113094, + "num_tokens": 3961842.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.182991920411586, + "rewards/margins": 19.472245895862578, + "rewards/rejected": -15.289253902435302, + "step": 430 + }, + { + "entropy": 1.2799105122685432, + "epoch": 1.3022222222222222, + "grad_norm": 0.00012683868408203125, + "learning_rate": 3.488441899896217e-05, + "logits/chosen": 2.345937521884795, + "logits/rejected": -1.2320921962004756, + "logps/chosen": -58.358681523799895, + "logps/rejected": -320.94243869781496, + "loss": 2.9142654511815635e-07, + "mean_token_accuracy": 0.679117338731885, + "num_tokens": 4051000.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.320470458269119, + "rewards/margins": 19.39275724887848, + "rewards/rejected": -15.072286760807037, + "step": 440 + }, + { + "entropy": 1.2606690004467964, + "epoch": 1.3318518518518518, + "grad_norm": 3.933906555175781e-05, + "learning_rate": 3.4087699264095745e-05, + "logits/chosen": 2.3590029943979434, + "logits/rejected": -1.180752685047298, + "logps/chosen": -57.81958304643631, + "logps/rejected": -327.3783897399902, + "loss": 1.31027945826645e-07, + "mean_token_accuracy": 0.6829047620296478, + "num_tokens": 4144901.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.355239224433899, + "rewards/margins": 19.49090631008148, + "rewards/rejected": -15.135667026042938, + "step": 450 + }, + { + "entropy": 1.2720958605408668, + "epoch": 1.3614814814814815, + "grad_norm": 7.152557373046875e-06, + "learning_rate": 3.328019698942826e-05, + "logits/chosen": 2.28385155999331, + "logits/rejected": -1.0655708622856899, + "logps/chosen": -65.57811061143875, + "logps/rejected": -324.391535949707, + "loss": 1.4767925904379808e-07, + "mean_token_accuracy": 0.6687033347785473, + "num_tokens": 4235782.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.3411215707659725, + "rewards/margins": 19.290599715709686, + "rewards/rejected": -14.949478232860566, + "step": 460 + }, + { + "entropy": 1.2874604403972625, + "epoch": 1.3911111111111112, + "grad_norm": 0.00011920928955078125, + "learning_rate": 3.246287027504237e-05, + "logits/chosen": 2.2650682684340646, + "logits/rejected": -1.1018559007875284, + "logps/chosen": -63.989255738258365, + "logps/rejected": -333.0100872039795, + "loss": 2.971911953864037e-07, + "mean_token_accuracy": 0.6702111043035984, + "num_tokens": 4327602.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.187621220946312, + "rewards/margins": 19.38519823551178, + "rewards/rejected": -15.197577047348023, + "step": 470 + }, + { + "entropy": 1.293594116717577, + "epoch": 1.4207407407407406, + "grad_norm": 9.870529174804688e-05, + "learning_rate": 3.163668887770181e-05, + "logits/chosen": 2.306861956625707, + "logits/rejected": -1.1661940995045186, + "logps/chosen": -61.81727409362793, + "logps/rejected": -332.9074993133545, + "loss": 2.5863685095828257e-07, + "mean_token_accuracy": 0.6736553743481636, + "num_tokens": 4419888.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.17706933170557, + "rewards/margins": 19.331134915351868, + "rewards/rejected": -15.154065454006195, + "step": 480 + }, + { + "entropy": 1.2759695634245873, + "epoch": 1.4503703703703703, + "grad_norm": 6.771087646484375e-05, + "learning_rate": 3.08026330602367e-05, + "logits/chosen": 2.338572971372103, + "logits/rejected": -1.1794468686720447, + "logps/chosen": -58.53629996776581, + "logps/rejected": -332.7186403274536, + "loss": 8.574451726417465e-08, + "mean_token_accuracy": 0.6770360998809337, + "num_tokens": 4511414.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.380688160657883, + "rewards/margins": 19.57960090637207, + "rewards/rejected": -15.198912739753723, + "step": 490 + }, + { + "entropy": 1.2591664418578148, + "epoch": 1.48, + "grad_norm": 8.046627044677734e-06, + "learning_rate": 2.9961692428463285e-05, + "logits/chosen": 2.2509338882701817, + "logits/rejected": -1.136340958920119, + "logps/chosen": -66.00177557468415, + "logps/rejected": -332.10723934173586, + "loss": 3.2082580219139346e-07, + "mean_token_accuracy": 0.6653452910482883, + "num_tokens": 4604818.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.144159619510174, + "rewards/margins": 19.529365801811217, + "rewards/rejected": -15.385206091403962, + "step": 500 + }, + { + "epoch": 1.48, + "eval_entropy": 1.2810596311009013, + "eval_logits/chosen": 2.2449733761384874, + "eval_logits/rejected": -1.1597314363523379, + "eval_logps/chosen": -67.92375413854639, + "eval_logps/rejected": -326.1967938829969, + "eval_loss": 7.327445018745493e-06, + "eval_mean_token_accuracy": 0.6633227342492217, + "eval_num_tokens": 4604818.0, + "eval_rewards/accuracies": 1.0, + "eval_rewards/chosen": 4.322750580894364, + "eval_rewards/margins": 19.43744218599546, + "eval_rewards/rejected": -15.114691607601992, + "eval_runtime": 38.6636, + "eval_samples_per_second": 7.371, + "eval_steps_per_second": 3.699, + "step": 500 + }, + { + "entropy": 1.2616984777152538, + "epoch": 1.5096296296296297, + "grad_norm": 0.000125885009765625, + "learning_rate": 2.9114864757018352e-05, + "logits/chosen": 2.294192165543051, + "logits/rejected": -1.0944581394998258, + "logps/chosen": -64.26435973644257, + "logps/rejected": -325.8657173156738, + "loss": 1.0070654070659657e-07, + "mean_token_accuracy": 0.6660138607025147, + "num_tokens": 4695387.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.35933136343956, + "rewards/margins": 19.695837664604188, + "rewards/rejected": -15.336506283283233, + "step": 510 + }, + { + "entropy": 1.2706036016345024, + "epoch": 1.5392592592592593, + "grad_norm": 8.821487426757812e-05, + "learning_rate": 2.8263154805501297e-05, + "logits/chosen": 2.228209829000261, + "logits/rejected": -1.113362437458028, + "logps/chosen": -69.59165489673615, + "logps/rejected": -321.3457368850708, + "loss": 1.4221999435903854e-07, + "mean_token_accuracy": 0.6667736105620861, + "num_tokens": 4788711.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.314350298047065, + "rewards/margins": 19.27485271692276, + "rewards/rejected": -14.960502445697784, + "step": 520 + }, + { + "entropy": 1.2767572030425072, + "epoch": 1.568888888888889, + "grad_norm": 0.0002460479736328125, + "learning_rate": 2.7407573126328545e-05, + "logits/chosen": 2.3174187011624654, + "logits/rejected": -1.2211792118899774, + "logps/chosen": -58.480189538002016, + "logps/rejected": -321.1500442504883, + "loss": 1.383585185976699e-07, + "mean_token_accuracy": 0.675158528238535, + "num_tokens": 4877592.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.162171944975853, + "rewards/margins": 19.111147022247316, + "rewards/rejected": -14.948975014686585, + "step": 530 + }, + { + "entropy": 1.2621485769748688, + "epoch": 1.5985185185185187, + "grad_norm": 3.844499588012695e-06, + "learning_rate": 2.6549134865714868e-05, + "logits/chosen": 2.26315295877127, + "logits/rejected": -1.12053805020359, + "logps/chosen": -64.27606203556061, + "logps/rejected": -328.73465633392334, + "loss": 1.7509564713691362e-07, + "mean_token_accuracy": 0.670067022740841, + "num_tokens": 4969700.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.281642350554466, + "rewards/margins": 19.606130039691926, + "rewards/rejected": -15.32448751926422, + "step": 540 + }, + { + "entropy": 1.266157256066799, + "epoch": 1.6281481481481481, + "grad_norm": 1.4901161193847656e-05, + "learning_rate": 2.5688858559204053e-05, + "logits/chosen": 2.3036601533908074, + "logits/rejected": -1.1204950201201778, + "logps/chosen": -63.52790377140045, + "logps/rejected": -332.37768840789795, + "loss": 1.9515678104653488e-07, + "mean_token_accuracy": 0.6726867496967316, + "num_tokens": 5064766.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.242260693013668, + "rewards/margins": 19.574455058574678, + "rewards/rejected": -15.33219426870346, + "step": 550 + }, + { + "entropy": 1.272486585378647, + "epoch": 1.6577777777777778, + "grad_norm": 3.933906555175781e-06, + "learning_rate": 2.4827764923178246e-05, + "logits/chosen": 2.3149282625691305, + "logits/rejected": -1.2307484890552138, + "logps/chosen": -60.608607578277585, + "logps/rejected": -325.5159580230713, + "loss": 1.471338123337773e-07, + "mean_token_accuracy": 0.6735777668654919, + "num_tokens": 5155008.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.4290194630622866, + "rewards/margins": 19.601059877872466, + "rewards/rejected": -15.172040581703186, + "step": 560 + }, + { + "entropy": 1.2813369497656821, + "epoch": 1.6874074074074072, + "grad_norm": 0.000339508056640625, + "learning_rate": 2.3966875643779667e-05, + "logits/chosen": 2.2549751619675136, + "logits/rejected": -1.1594325055530381, + "logps/chosen": -64.34185855388641, + "logps/rejected": -327.56716537475586, + "loss": 1.8051871393254259e-07, + "mean_token_accuracy": 0.6600819997489452, + "num_tokens": 5248360.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.4127004653215405, + "rewards/margins": 19.473870050907134, + "rewards/rejected": -15.061169505119324, + "step": 570 + }, + { + "entropy": 1.2690633878111839, + "epoch": 1.717037037037037, + "grad_norm": 0.000118255615234375, + "learning_rate": 2.3107212164681776e-05, + "logits/chosen": 2.277724768747875, + "logits/rejected": -1.1787549020085861, + "logps/chosen": -66.66345942020416, + "logps/rejected": -323.8808866500855, + "loss": 2.298829940627911e-07, + "mean_token_accuracy": 0.6669816076755524, + "num_tokens": 5340554.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.2985071912407875, + "rewards/margins": 19.356661868095397, + "rewards/rejected": -15.058154690265656, + "step": 580 + }, + { + "entropy": 1.303315782546997, + "epoch": 1.7466666666666666, + "grad_norm": 9.894371032714844e-06, + "learning_rate": 2.224979447514802e-05, + "logits/chosen": 2.2417929976862743, + "logits/rejected": -1.1242295347690896, + "logps/chosen": -66.85722150802613, + "logps/rejected": -326.75849361419677, + "loss": 5.121967660670635e-07, + "mean_token_accuracy": 0.6579514071345329, + "num_tokens": 5433019.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.279269021749497, + "rewards/margins": 19.181958985328674, + "rewards/rejected": -14.902689921855927, + "step": 590 + }, + { + "entropy": 1.2511712044477463, + "epoch": 1.7762962962962963, + "grad_norm": 1.811981201171875e-05, + "learning_rate": 2.1395639899816333e-05, + "logits/chosen": 2.4288169740201644, + "logits/rejected": -1.0983362881453427, + "logps/chosen": -58.963488447666165, + "logps/rejected": -328.273935508728, + "loss": 5.187651368032676e-07, + "mean_token_accuracy": 0.6844911381602288, + "num_tokens": 5527564.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.607324048876762, + "rewards/margins": 19.77645412683487, + "rewards/rejected": -15.169130110740662, + "step": 600 + }, + { + "epoch": 1.7762962962962963, + "eval_entropy": 1.2805373877078503, + "eval_logits/chosen": 2.243851235815972, + "eval_logits/rejected": -1.1594670058021077, + "eval_logps/chosen": -67.99770445923706, + "eval_logps/rejected": -326.55811144421983, + "eval_loss": 6.923921773704933e-06, + "eval_mean_token_accuracy": 0.6634519100189209, + "eval_num_tokens": 5527564.0, + "eval_rewards/accuracies": 1.0, + "eval_rewards/chosen": 4.315355520148377, + "eval_rewards/margins": 19.466178794007202, + "eval_rewards/rejected": -15.15082325968709, + "eval_runtime": 39.0736, + "eval_samples_per_second": 7.294, + "eval_steps_per_second": 3.66, + "step": 600 + }, + { + "entropy": 1.2721578769385815, + "epoch": 1.805925925925926, + "grad_norm": 1.3053417205810547e-05, + "learning_rate": 2.054576189164518e-05, + "logits/chosen": 2.3202715445664515, + "logits/rejected": -1.132963813611814, + "logps/chosen": -60.805744123458865, + "logps/rejected": -332.585825920105, + "loss": 3.902006255884771e-08, + "mean_token_accuracy": 0.6726082235574722, + "num_tokens": 5620386.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.585019749403, + "rewards/margins": 20.017272901535033, + "rewards/rejected": -15.43225302696228, + "step": 610 + }, + { + "entropy": 1.2652241297066211, + "epoch": 1.8355555555555556, + "grad_norm": 1.633167266845703e-05, + "learning_rate": 1.9701168829453306e-05, + "logits/chosen": 2.259219360616968, + "logits/rejected": -1.0503303018419232, + "logps/chosen": -66.24489846229554, + "logps/rejected": -322.5876424789429, + "loss": 1.7130523701780476e-07, + "mean_token_accuracy": 0.6619290798902512, + "num_tokens": 5712637.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.140004143118858, + "rewards/margins": 19.201012241840363, + "rewards/rejected": -15.06100798845291, + "step": 620 + }, + { + "entropy": 1.2682488977909088, + "epoch": 1.8651851851851853, + "grad_norm": 2.956390380859375e-05, + "learning_rate": 1.8862862821480025e-05, + "logits/chosen": 2.3071657025101677, + "logits/rejected": -1.1965660990092328, + "logps/chosen": -65.7512573480606, + "logps/rejected": -326.63800010681155, + "loss": 9.80972572506289e-08, + "mean_token_accuracy": 0.6676984764635563, + "num_tokens": 5804212.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.233162444829941, + "rewards/margins": 19.35871180295944, + "rewards/rejected": -15.125549244880677, + "step": 630 + }, + { + "entropy": 1.2738682702183723, + "epoch": 1.894814814814815, + "grad_norm": 1.3709068298339844e-06, + "learning_rate": 1.8031838516385423e-05, + "logits/chosen": 2.277379836690332, + "logits/rejected": -1.1057443809996192, + "logps/chosen": -64.37387623786927, + "logps/rejected": -329.1884712219238, + "loss": 5.206376272326452e-08, + "mean_token_accuracy": 0.6642427556216717, + "num_tokens": 5897138.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.30352343916893, + "rewards/margins": 19.877286291122438, + "rewards/rejected": -15.573762822151185, + "step": 640 + }, + { + "entropy": 1.2760708816349506, + "epoch": 1.9244444444444444, + "grad_norm": 5.1021575927734375e-05, + "learning_rate": 1.7209081923101472e-05, + "logits/chosen": 2.327136815796652, + "logits/rejected": -1.1425043440165608, + "logps/chosen": -66.02759101390839, + "logps/rejected": -332.0861042022705, + "loss": 1.4513241239910712e-07, + "mean_token_accuracy": 0.66774348244071, + "num_tokens": 5990180.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.311979895830154, + "rewards/margins": 19.720207822322845, + "rewards/rejected": -15.408227932453155, + "step": 650 + }, + { + "entropy": 1.2696824759244918, + "epoch": 1.954074074074074, + "grad_norm": 2.4557113647460938e-05, + "learning_rate": 1.639556924093404e-05, + "logits/chosen": 2.2885721283972242, + "logits/rejected": -1.12765834632216, + "logps/chosen": -64.96237716674804, + "logps/rejected": -326.4824584960937, + "loss": 1.2898549357487353e-07, + "mean_token_accuracy": 0.6572523653507233, + "num_tokens": 6084186.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.2255510374903675, + "rewards/margins": 19.46041429042816, + "rewards/rejected": -15.234863257408142, + "step": 660 + }, + { + "entropy": 1.2641157731413841, + "epoch": 1.9837037037037037, + "grad_norm": 1.9073486328125e-06, + "learning_rate": 1.5592265701304114e-05, + "logits/chosen": 2.3036376802600183, + "logits/rejected": -1.1734117772095596, + "logps/chosen": -65.30954070091248, + "logps/rejected": -323.9220626831055, + "loss": 2.2752515178581233e-07, + "mean_token_accuracy": 0.6635996960103512, + "num_tokens": 6177151.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.288326196372509, + "rewards/margins": 19.406356739997864, + "rewards/rejected": -15.118030607700348, + "step": 670 + }, + { + "entropy": 1.2779945597836846, + "epoch": 2.011851851851852, + "grad_norm": 0.000255584716796875, + "learning_rate": 1.4800124422502334e-05, + "logits/chosen": 2.3972204176263543, + "logits/rejected": -1.1209583839266675, + "logps/chosen": -61.41229806448284, + "logps/rejected": -324.14299693860505, + "loss": 3.6133037610852626e-07, + "mean_token_accuracy": 0.6630792186448449, + "num_tokens": 6262744.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.139316172976243, + "rewards/margins": 19.215763267717865, + "rewards/rejected": -15.076447160620438, + "step": 680 + }, + { + "entropy": 1.266596408188343, + "epoch": 2.0414814814814815, + "grad_norm": 4.4889748096466064e-07, + "learning_rate": 1.4020085278815745e-05, + "logits/chosen": 2.3154754788175156, + "logits/rejected": -1.152066471886049, + "logps/chosen": -62.28092246055603, + "logps/rejected": -331.86610050201415, + "loss": 7.074050927258213e-08, + "mean_token_accuracy": 0.6724916785955429, + "num_tokens": 6357968.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.411373123526573, + "rewards/margins": 19.874622988700867, + "rewards/rejected": -15.4632497549057, + "step": 690 + }, + { + "entropy": 1.2671823538839817, + "epoch": 2.071111111111111, + "grad_norm": 0.0002994537353515625, + "learning_rate": 1.3253073785368547e-05, + "logits/chosen": 2.29187693793979, + "logits/rejected": -1.1263780908242413, + "logps/chosen": -65.86139830350876, + "logps/rejected": -331.38874130249025, + "loss": 2.163802719223895e-07, + "mean_token_accuracy": 0.6635798320174218, + "num_tokens": 6449816.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.283302050828934, + "rewards/margins": 19.748378670215608, + "rewards/rejected": -15.465076541900634, + "step": 700 + }, + { + "epoch": 2.071111111111111, + "eval_entropy": 1.2803548124286679, + "eval_logits/chosen": 2.243403260803776, + "eval_logits/rejected": -1.1587966303436696, + "eval_logps/chosen": -68.02522693980823, + "eval_logps/rejected": -326.6569073016827, + "eval_loss": 7.350444775511278e-06, + "eval_mean_token_accuracy": 0.6639135730016482, + "eval_num_tokens": 6449816.0, + "eval_rewards/accuracies": 1.0, + "eval_rewards/chosen": 4.312603289847607, + "eval_rewards/margins": 19.473306129028746, + "eval_rewards/rejected": -15.160702772073812, + "eval_runtime": 39.1309, + "eval_samples_per_second": 7.283, + "eval_steps_per_second": 3.654, + "step": 700 + }, + { + "entropy": 1.2728232815861702, + "epoch": 2.100740740740741, + "grad_norm": 8.821487426757812e-05, + "learning_rate": 1.2500000000000006e-05, + "logits/chosen": 2.255679331290835, + "logits/rejected": -1.1216202176026864, + "logps/chosen": -61.825205373764035, + "logps/rejected": -323.78655910491943, + "loss": 2.141799996024929e-07, + "mean_token_accuracy": 0.667589108645916, + "num_tokens": 6540123.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.437801697850228, + "rewards/margins": 19.34732162952423, + "rewards/rejected": -14.90951988697052, + "step": 710 + }, + { + "entropy": 1.271144437789917, + "epoch": 2.1303703703703705, + "grad_norm": 6.771087646484375e-05, + "learning_rate": 1.1761757443482286e-05, + "logits/chosen": 2.2690192203566064, + "logits/rejected": -1.176060724310163, + "logps/chosen": -65.25097322463989, + "logps/rejected": -325.64191341400146, + "loss": 7.844523679523263e-08, + "mean_token_accuracy": 0.6650686420500278, + "num_tokens": 6629998.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.375448957085609, + "rewards/margins": 19.631182181835175, + "rewards/rejected": -15.255733144283294, + "step": 720 + }, + { + "entropy": 1.263370680809021, + "epoch": 2.16, + "grad_norm": 0.00067901611328125, + "learning_rate": 1.1039222039359645e-05, + "logits/chosen": 2.2999157883359946, + "logits/rejected": -1.1241040776625622, + "logps/chosen": -65.3091173171997, + "logps/rejected": -327.6589548110962, + "loss": 6.326518359855982e-07, + "mean_token_accuracy": 0.6666021130979061, + "num_tokens": 6721579.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.218965843319893, + "rewards/margins": 19.43423910140991, + "rewards/rejected": -15.21527328491211, + "step": 730 + }, + { + "entropy": 1.2835220411419868, + "epoch": 2.18962962962963, + "grad_norm": 9.059906005859375e-06, + "learning_rate": 1.0333251074666609e-05, + "logits/chosen": 2.2917566691940756, + "logits/rejected": -1.2816091428578473, + "logps/chosen": -66.77954509258271, + "logps/rejected": -333.47667198181153, + "loss": 1.7912802832142916e-07, + "mean_token_accuracy": 0.6699437476694584, + "num_tokens": 6813377.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.310945153236389, + "rewards/margins": 19.620563447475433, + "rewards/rejected": -15.309618008136749, + "step": 740 + }, + { + "entropy": 1.2834552891552449, + "epoch": 2.2192592592592595, + "grad_norm": 2.8252601623535156e-05, + "learning_rate": 9.644682182758306e-06, + "logits/chosen": 2.296253967390375, + "logits/rejected": -1.1382098734020025, + "logps/chosen": -65.18716852664947, + "logps/rejected": -333.20018062591555, + "loss": 3.619171593527426e-07, + "mean_token_accuracy": 0.670500548928976, + "num_tokens": 6906265.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.34758489727974, + "rewards/margins": 19.480838096141817, + "rewards/rejected": -15.133253240585328, + "step": 750 + }, + { + "entropy": 1.2889503717422486, + "epoch": 2.2488888888888887, + "grad_norm": 3.6716461181640625e-05, + "learning_rate": 8.974332349459994e-06, + "logits/chosen": 2.274084705882701, + "logits/rejected": -1.1973438289074938, + "logps/chosen": -62.72564001083374, + "logps/rejected": -335.81445999145507, + "loss": 2.772670995909721e-07, + "mean_token_accuracy": 0.663724122941494, + "num_tokens": 7004404.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.22343917787075, + "rewards/margins": 19.607576775550843, + "rewards/rejected": -15.384137523174285, + "step": 760 + }, + { + "entropy": 1.2709729745984077, + "epoch": 2.2785185185185184, + "grad_norm": 6.818771362304688e-05, + "learning_rate": 8.322996943714672e-06, + "logits/chosen": 2.315020811648067, + "logits/rejected": -1.1844662490433007, + "logps/chosen": -60.06333501338959, + "logps/rejected": -324.07994575500487, + "loss": 6.899633149259899e-08, + "mean_token_accuracy": 0.674921452999115, + "num_tokens": 7091630.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.219157862663269, + "rewards/margins": 19.409486842155456, + "rewards/rejected": -15.190328931808471, + "step": 770 + }, + { + "entropy": 1.2802273795008658, + "epoch": 2.308148148148148, + "grad_norm": 7.677078247070312e-05, + "learning_rate": 7.691448773879257e-06, + "logits/chosen": 2.25499691013824, + "logits/rejected": -1.1512603749381776, + "logps/chosen": -63.52460696697235, + "logps/rejected": -325.2589776992798, + "loss": 1.4859768953101594e-07, + "mean_token_accuracy": 0.6620502531528473, + "num_tokens": 7182707.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.2716171652078625, + "rewards/margins": 19.37758446931839, + "rewards/rejected": -15.10596705675125, + "step": 780 + }, + { + "entropy": 1.2827171966433526, + "epoch": 2.3377777777777777, + "grad_norm": 6.645917892456055e-06, + "learning_rate": 7.080437170788723e-06, + "logits/chosen": 2.311279925102533, + "logits/rejected": -1.1746961520146812, + "logps/chosen": -59.46050326824188, + "logps/rejected": -336.5900556564331, + "loss": 3.455784508332727e-07, + "mean_token_accuracy": 0.6743544064462185, + "num_tokens": 7275656.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.199921746551991, + "rewards/margins": 19.863027906417848, + "rewards/rejected": -15.663106298446655, + "step": 790 + }, + { + "entropy": 1.2675680316984654, + "epoch": 2.3674074074074074, + "grad_norm": 0.00015926361083984375, + "learning_rate": 6.490687098676332e-06, + "logits/chosen": 2.3366430778544096, + "logits/rejected": -1.1229084532499813, + "logps/chosen": -58.34785094261169, + "logps/rejected": -331.7050970077515, + "loss": 6.806028522987617e-07, + "mean_token_accuracy": 0.6765504986047745, + "num_tokens": 7367211.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.387070021033287, + "rewards/margins": 19.79595556259155, + "rewards/rejected": -15.40888546705246, + "step": 800 + }, + { + "epoch": 2.3674074074074074, + "eval_entropy": 1.2803315270197142, + "eval_logits/chosen": 2.2436578263578553, + "eval_logits/rejected": -1.158513689404391, + "eval_logps/chosen": -68.02018330980847, + "eval_logps/rejected": -326.71273963768164, + "eval_loss": 7.024308615655173e-06, + "eval_mean_token_accuracy": 0.6631912376497175, + "eval_num_tokens": 7367211.0, + "eval_rewards/accuracies": 1.0, + "eval_rewards/chosen": 4.313107631006441, + "eval_rewards/margins": 19.479393625592852, + "eval_rewards/rejected": -15.166285921643665, + "eval_runtime": 39.4139, + "eval_samples_per_second": 7.231, + "eval_steps_per_second": 3.628, + "step": 800 + }, + { + "entropy": 1.2536483012139796, + "epoch": 2.397037037037037, + "grad_norm": 0.00010061264038085938, + "learning_rate": 5.9228982950048416e-06, + "logits/chosen": 2.306488303700283, + "logits/rejected": -1.102614074804673, + "logps/chosen": -64.7256872177124, + "logps/rejected": -316.81419982910154, + "loss": 2.7541559575183785e-07, + "mean_token_accuracy": 0.6695533588528633, + "num_tokens": 7456110.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.3090651541948315, + "rewards/margins": 19.122487926483153, + "rewards/rejected": -14.813422644138337, + "step": 810 + }, + { + "entropy": 1.2691614858806133, + "epoch": 2.4266666666666667, + "grad_norm": 3.844499588012695e-06, + "learning_rate": 5.377744440229135e-06, + "logits/chosen": 2.276143363869749, + "logits/rejected": -1.2205610514710212, + "logps/chosen": -64.77891628742218, + "logps/rejected": -330.2338497161865, + "loss": 9.86900658972445e-08, + "mean_token_accuracy": 0.6663463093340397, + "num_tokens": 7548949.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.394037923216819, + "rewards/margins": 19.882170844078065, + "rewards/rejected": -15.488132905960082, + "step": 820 + }, + { + "entropy": 1.2759164057672023, + "epoch": 2.4562962962962964, + "grad_norm": 1.5735626220703125e-05, + "learning_rate": 4.855872358475547e-06, + "logits/chosen": 2.2611198054529975, + "logits/rejected": -1.1668456534925964, + "logps/chosen": -63.698548150062564, + "logps/rejected": -323.2269037246704, + "loss": 1.5968792013154598e-07, + "mean_token_accuracy": 0.6731547065079212, + "num_tokens": 7637248.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.312624426186085, + "rewards/margins": 19.461497557163238, + "rewards/rejected": -15.148873174190522, + "step": 830 + }, + { + "entropy": 1.2611838012933732, + "epoch": 2.485925925925926, + "grad_norm": 0.0002841949462890625, + "learning_rate": 4.357901250086108e-06, + "logits/chosen": 2.2900714239429774, + "logits/rejected": -1.2470973023927523, + "logps/chosen": -62.78718073368073, + "logps/rejected": -324.9568237304687, + "loss": 1.4255396081352955e-07, + "mean_token_accuracy": 0.6746141783893108, + "num_tokens": 7726306.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.291703987121582, + "rewards/margins": 19.421786904335022, + "rewards/rejected": -15.130082988739014, + "step": 840 + }, + { + "entropy": 1.2697825357317924, + "epoch": 2.5155555555555553, + "grad_norm": 0.0002613067626953125, + "learning_rate": 3.884421956938378e-06, + "logits/chosen": 2.3862500773086817, + "logits/rejected": -1.1250058733609287, + "logps/chosen": -61.92496125698089, + "logps/rejected": -327.44327507019045, + "loss": 1.493016384301882e-07, + "mean_token_accuracy": 0.6789508298039436, + "num_tokens": 7822125.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.364183455705643, + "rewards/margins": 19.506937170028685, + "rewards/rejected": -15.142753863334656, + "step": 850 + }, + { + "entropy": 1.2504698053002357, + "epoch": 2.5451851851851854, + "grad_norm": 4.506111145019531e-05, + "learning_rate": 3.435996261412591e-06, + "logits/chosen": 2.2986260733744173, + "logits/rejected": -1.1058551351474077, + "logps/chosen": -65.89442076683045, + "logps/rejected": -330.00583362579346, + "loss": 1.4223222706277737e-07, + "mean_token_accuracy": 0.6768516503274441, + "num_tokens": 7915620.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.385465979576111, + "rewards/margins": 20.04776966571808, + "rewards/rejected": -15.662303495407105, + "step": 860 + }, + { + "entropy": 1.2795700900256635, + "epoch": 2.5748148148148147, + "grad_norm": 2.5153160095214844e-05, + "learning_rate": 3.013156219837776e-06, + "logits/chosen": 2.237134730489094, + "logits/rejected": -1.1218436377455188, + "logps/chosen": -65.73119521141052, + "logps/rejected": -322.3418972015381, + "loss": 2.536130750740995e-07, + "mean_token_accuracy": 0.665914348512888, + "num_tokens": 8006742.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.355473381280899, + "rewards/margins": 19.215320122241973, + "rewards/rejected": -14.85984662771225, + "step": 870 + }, + { + "entropy": 1.2761830180883407, + "epoch": 2.6044444444444443, + "grad_norm": 1.1742115020751953e-05, + "learning_rate": 2.6164035312078448e-06, + "logits/chosen": 2.2800606568463015, + "logits/rejected": -1.1315629358769943, + "logps/chosen": -61.99486291408539, + "logps/rejected": -326.9346130371094, + "loss": 2.3778172817401355e-07, + "mean_token_accuracy": 0.6662385709583759, + "num_tokens": 8100018.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.3306570738554, + "rewards/margins": 19.479354202747345, + "rewards/rejected": -15.148697006702424, + "step": 880 + }, + { + "entropy": 1.279340496659279, + "epoch": 2.634074074074074, + "grad_norm": 9.632110595703125e-05, + "learning_rate": 2.2462089419165777e-06, + "logits/chosen": 2.3029893365635097, + "logits/rejected": -1.171205228496424, + "logps/chosen": -65.00987663269044, + "logps/rejected": -330.74864501953124, + "loss": 6.256735787246726e-08, + "mean_token_accuracy": 0.6665534779429436, + "num_tokens": 8194303.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.474057069420814, + "rewards/margins": 19.85085200071335, + "rewards/rejected": -15.376794970035553, + "step": 890 + }, + { + "entropy": 1.2603400394320488, + "epoch": 2.6637037037037037, + "grad_norm": 2.4557113647460938e-05, + "learning_rate": 1.9030116872178316e-06, + "logits/chosen": 2.249223475648134, + "logits/rejected": -1.0936998011992798, + "logps/chosen": -64.42266466617585, + "logps/rejected": -319.57002086639403, + "loss": 1.4027624501977697e-07, + "mean_token_accuracy": 0.6666488841176033, + "num_tokens": 8285684.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.301889230310917, + "rewards/margins": 19.297891116142274, + "rewards/rejected": -14.99600180387497, + "step": 900 + }, + { + "epoch": 2.6637037037037037, + "eval_entropy": 1.2802694969243937, + "eval_logits/chosen": 2.2445470870400355, + "eval_logits/rejected": -1.159552092885801, + "eval_logps/chosen": -67.97919230694538, + "eval_logps/rejected": -326.7502327232094, + "eval_loss": 6.49735875413171e-06, + "eval_mean_token_accuracy": 0.6639414579718264, + "eval_num_tokens": 8285684.0, + "eval_rewards/accuracies": 1.0, + "eval_rewards/chosen": 4.317206760803303, + "eval_rewards/margins": 19.48724203843337, + "eval_rewards/rejected": -15.17003523553168, + "eval_runtime": 39.451, + "eval_samples_per_second": 7.224, + "eval_steps_per_second": 3.625, + "step": 900 + } + ], + "logging_steps": 10, + "max_steps": 1013, + "num_input_tokens_seen": 0, + "num_train_epochs": 3, + "save_steps": 100, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 4.3836356641429094e+17, + "train_batch_size": 2, + "trial_name": null, + "trial_params": null +} diff --git a/checkpoint-900/training_args.bin b/checkpoint-900/training_args.bin new file mode 100644 index 0000000..5ac503c --- /dev/null +++ b/checkpoint-900/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9511438415c2456684b2907b1d3a2888ce92542e75957d8ee17f0b9dda11c01a +size 7249 diff --git a/config.json b/config.json new file mode 100644 index 0000000..6b475aa --- /dev/null +++ b/config.json @@ -0,0 +1,62 @@ +{ + "architectures": [ + "Qwen2ForCausalLM" + ], + "attention_dropout": 0.0, + "bos_token_id": null, + "dtype": "bfloat16", + "eos_token_id": 151643, + "hidden_act": "silu", + "hidden_size": 3584, + "initializer_range": 0.02, + "intermediate_size": 18944, + "layer_types": [ + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention" + ], + "max_position_embeddings": 131072, + "max_window_layers": 28, + "model_type": "qwen2", + "num_attention_heads": 28, + "num_hidden_layers": 28, + "num_key_value_heads": 4, + "pad_token_id": 151643, + "rms_norm_eps": 1e-06, + "rope_parameters": { + "rope_theta": 1000000.0, + "rope_type": "default" + }, + "sliding_window": null, + "tie_word_embeddings": false, + "transformers_version": "5.5.4", + "use_cache": false, + "use_mrope": false, + "use_sliding_window": false, + "vocab_size": 152064 +} diff --git a/debug.log b/debug.log new file mode 100644 index 0000000..4829eac --- /dev/null +++ b/debug.log @@ -0,0 +1,3097 @@ +[2026-04-16 10:31:52,083] [DEBUG] [axolotl.utils.config.log_gpu_memory_usage:127] [PID:135] baseline 0.000GB () +[2026-04-16 10:31:52,086] [INFO] [axolotl.cli.config.load_cfg:341] [PID:135] config: +{ + "activation_offloading": false, + "axolotl_config_path": "./config.yaml", + "base_model": "Qwen/Qwen2.5-7B", + "base_model_config": "Qwen/Qwen2.5-7B", + "batch_size": 16, + "bf16": true, + "capabilities": { + "bf16": true, + "compute_capability": "sm_80", + "fp8": false, + "n_gpu": 1, + "n_node": 1, + "tf32": true + }, + "context_parallel_size": 1, + "dataloader_num_workers": 1, + "dataloader_pin_memory": true, + "dataloader_prefetch_factor": 256, + "dataset_num_proc": 96, + "dataset_prepared_path": "./prepared_data", + "datasets": [ + { + "path": "Bossssss/medqa-dpo-idk", + "split": "train", + "type": "passthrough" + } + ], + "ddp": false, + "device": "cuda:0", + "dion_rank_fraction": 1.0, + "dion_rank_multiple_of": 1, + "eaft_alpha": 1.0, + "eaft_k": 20, + "env_capabilities": { + "torch_version": "2.9.1" + }, + "eval_batch_size": 2, + "eval_causal_lm_metrics": [ + "sacrebleu", + "comet", + "ter", + "chrf" + ], + "eval_max_new_tokens": 128, + "eval_steps": 100, + "eval_strategy": "steps", + "eval_table_size": 0, + "experimental_skip_move_to_device": true, + "flash_attention": true, + "fp16": false, + "generate_samples": false, + "generation_do_sample": true, + "generation_max_new_tokens": 50, + "generation_prompt_ratio": 0.5, + "generation_temperature": 0.7, + "gradient_accumulation_steps": 8, + "gradient_checkpointing": true, + "greater_is_better": false, + "include_tkps": true, + "is_falcon_derived_model": false, + "is_llama_derived_model": false, + "is_mistral_derived_model": false, + "layer_offloading": false, + "learning_rate": 5e-05, + "lisa_layers_attribute": "model.layers", + "load_best_model_at_end": true, + "load_in_4bit": false, + "load_in_8bit": false, + "local_rank": 0, + "logging_steps": 10, + "lora_dropout": 0.0, + "loraplus_lr_embedding": 1e-06, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "mean_resizing_embeddings": false, + "merge_method": "memory_efficient", + "metric_for_best_model": "eval_loss", + "micro_batch_size": 2, + "model_config_type": "qwen2", + "num_epochs": 3.0, + "num_generation_samples": 3, + "optimizer": "adamw_bnb_8bit", + "otel_metrics_host": "localhost", + "otel_metrics_port": 8000, + "output_dir": "kikiyaa/qwen-dpo-finetuned-ver2", + "pretrain_multipack_attn": true, + "profiler_steps_start": 0, + "qlora_sharded_model_loading": false, + "quantize_moe_experts": false, + "ray_num_workers": 1, + "resources_per_worker": { + "GPU": 1 + }, + "rl": "dpo", + "rl_beta": 0.1, + "sample_packing": false, + "sample_packing_bin_size": 200, + "sample_packing_group_size": 100000, + "save_only_model": false, + "save_safetensors": true, + "save_steps": 100, + "save_total_limit": 3, + "seed": 42, + "sequence_len": 2048, + "shuffle_before_merging_datasets": false, + "shuffle_merged_datasets": true, + "skip_prepare_dataset": false, + "streaming_multipack_buffer_size": 10000, + "strict": false, + "tensor_parallel_size": 1, + "tf32": true, + "tiled_mlp_use_original_mlp": true, + "tokenizer_config": "Qwen/Qwen2.5-7B", + "tokenizer_save_jinja_files": true, + "tokenizer_type": "AutoTokenizer", + "torch_dtype": "torch.bfloat16", + "train_on_inputs": false, + "trl": { + "async_prefetch": false, + "log_completions": false, + "mask_truncated_completions": false, + "ref_model_mixup_alpha": 0.9, + "ref_model_sync_steps": 64, + "replay_buffer_size": 0, + "replay_recompute_logps": true, + "reroll_max_groups": 1, + "reroll_start_fraction": 1.0, + "reward_num_workers": 1, + "scale_rewards": true, + "skip_zero_advantage_batches": true, + "sync_ref_model": false, + "use_data_producer": false, + "use_vllm": false, + "vllm_lora_sync": false, + "vllm_server_host": "0.0.0.0", + "vllm_server_port": 8000 + }, + "type_of_model": "AutoModelForCausalLM", + "use_otel_metrics": false, + "use_ray": false, + "val_set_size": 0.05, + "vllm": { + "device": "auto", + "dtype": "auto", + "gpu_memory_utilization": 0.9, + "host": "0.0.0.0", + "port": 8000 + }, + "warmup_ratio": 0.1, + "weight_decay": 0.01, + "world_size": 1 +} +[2026-04-16 10:31:57,826] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:307] [PID:135] EOS: 151643 / <|endoftext|> +[2026-04-16 10:31:57,826] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:308] [PID:135] BOS: None / None +[2026-04-16 10:31:57,826] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:309] [PID:135] PAD: 151643 / <|endoftext|> +[2026-04-16 10:31:57,826] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:310] [PID:135] UNK: None / None +[2026-04-16 10:31:57,831] [INFO] [axolotl.utils.data.shared.load_preprocessed_dataset:475] [PID:135] Loading prepared dataset from disk at prepared_data/b845853b1ffa0f92dd415602f19f08af... +[2026-04-16 10:31:57,902] [DEBUG] [axolotl.train.setup_model_and_tokenizer:70] [PID:135] loading tokenizer... Qwen/Qwen2.5-7B +[2026-04-16 10:31:59,556] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:307] [PID:135] EOS: 151643 / <|endoftext|> +[2026-04-16 10:31:59,556] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:308] [PID:135] BOS: None / None +[2026-04-16 10:31:59,556] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:309] [PID:135] PAD: 151643 / <|endoftext|> +[2026-04-16 10:31:59,556] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:310] [PID:135] UNK: None / None +[2026-04-16 10:31:59,556] [DEBUG] [axolotl.train.setup_model_and_tokenizer:81] [PID:135] Loading model +[2026-04-16 10:31:59,777] [DEBUG] [axolotl.monkeypatch.torchao_optim.patch_torchao_optim_state_8bit:75] [PID:135] Patched OptimState8bit for torch.compile compatibility +[2026-04-16 10:31:59,778] [DEBUG] [axolotl.monkeypatch.torchao_optim.patch_torchao_optim_state_8bit:122] [PID:135] Patched OptimState4bit for torch.compile compatibility +[2026-04-16 10:31:59,778] [DEBUG] [axolotl.monkeypatch.torchao_optim.patch_torchao_optim_state_8bit:154] [PID:135] Patched OptimStateFp8 for torch.compile compatibility +[2026-04-16 10:31:59,795] [DEBUG] [axolotl.monkeypatch.transformers.trainer_loss_calc.patch_evaluation_loop:94] [PID:135] Patched Trainer.evaluation_loop with nanmean loss calculation +[2026-04-16 10:31:59,798] [DEBUG] [axolotl.monkeypatch.transformers.trainer_loss_calc.patch_maybe_log_save_evaluate:148] [PID:135] Patched Trainer._maybe_log_save_evaluate with nanmean loss calculation +[2026-04-16 10:32:00,017] [INFO] [axolotl.loaders.patch_manager._apply_trl_trainer_utils_patches:852] [PID:135] Patched trl.trainer.utils with Triton selective_log_softmax and entropy_from_logits + Fetching 4 files: 0%| | 0/4 [00:00", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +}