From 237ea2a8afb5eedab7f2ddc225e7a75d47c1916f Mon Sep 17 00:00:00 2001 From: ModelHub XC Date: Sat, 25 Jul 2026 15:41:09 +0800 Subject: [PATCH] =?UTF-8?q?=E5=88=9D=E5=A7=8B=E5=8C=96=E9=A1=B9=E7=9B=AE?= =?UTF-8?q?=EF=BC=8C=E7=94=B1ModelHub=20XC=E7=A4=BE=E5=8C=BA=E6=8F=90?= =?UTF-8?q?=E4=BE=9B=E6=A8=A1=E5=9E=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Model: manucif/latamgpt-1b-dpo-vanilla Source: Original Platform --- .gitattributes | 36 + README.md | 68 + all_results.json | 20 + chat_template.jinja | 5 + config.json | 36 + generation_config.json | 12 + model.safetensors | 3 + tokenizer.json | 3 + tokenizer_config.json | 15 + train_results.json | 20 + trainer_state.json | 3888 ++++++++++++++++++++++++++++++++++++++++ training_args.bin | 3 + 12 files changed, 4109 insertions(+) create mode 100644 .gitattributes create mode 100644 README.md create mode 100644 all_results.json create mode 100644 chat_template.jinja create mode 100644 config.json create mode 100644 generation_config.json create mode 100644 model.safetensors create mode 100644 tokenizer.json create mode 100644 tokenizer_config.json create mode 100644 train_results.json create mode 100644 trainer_state.json create mode 100644 training_args.bin diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..52373fe --- /dev/null +++ b/.gitattributes @@ -0,0 +1,36 @@ +*.7z filter=lfs diff=lfs merge=lfs -text +*.arrow filter=lfs diff=lfs merge=lfs -text +*.bin filter=lfs diff=lfs merge=lfs -text +*.bz2 filter=lfs diff=lfs merge=lfs -text +*.ckpt filter=lfs diff=lfs merge=lfs -text +*.ftz filter=lfs diff=lfs merge=lfs -text +*.gz filter=lfs diff=lfs merge=lfs -text +*.h5 filter=lfs diff=lfs merge=lfs -text +*.joblib filter=lfs diff=lfs merge=lfs -text +*.lfs.* filter=lfs diff=lfs merge=lfs -text +*.mlmodel filter=lfs diff=lfs merge=lfs -text +*.model filter=lfs diff=lfs merge=lfs -text +*.msgpack filter=lfs diff=lfs merge=lfs -text +*.npy filter=lfs diff=lfs merge=lfs -text +*.npz filter=lfs diff=lfs merge=lfs -text +*.onnx filter=lfs diff=lfs merge=lfs -text +*.ot filter=lfs diff=lfs merge=lfs -text +*.parquet filter=lfs diff=lfs merge=lfs -text +*.pb filter=lfs diff=lfs merge=lfs -text +*.pickle filter=lfs diff=lfs merge=lfs -text +*.pkl filter=lfs diff=lfs merge=lfs -text +*.pt filter=lfs diff=lfs merge=lfs -text +*.pth filter=lfs diff=lfs merge=lfs -text +*.rar filter=lfs diff=lfs merge=lfs -text +*.safetensors filter=lfs diff=lfs merge=lfs -text +saved_model/**/* filter=lfs diff=lfs merge=lfs -text +*.tar.* filter=lfs diff=lfs merge=lfs -text +*.tar filter=lfs diff=lfs merge=lfs -text +*.tflite filter=lfs diff=lfs merge=lfs -text +*.tgz filter=lfs diff=lfs merge=lfs -text +*.wasm filter=lfs diff=lfs merge=lfs -text +*.xz filter=lfs diff=lfs merge=lfs -text +*.zip filter=lfs diff=lfs merge=lfs -text +*.zst filter=lfs diff=lfs merge=lfs -text +*tfevents* filter=lfs diff=lfs merge=lfs -text +tokenizer.json filter=lfs diff=lfs merge=lfs -text diff --git a/README.md b/README.md new file mode 100644 index 0000000..bd60266 --- /dev/null +++ b/README.md @@ -0,0 +1,68 @@ +--- +library_name: transformers +model_name: dpo_output_vanilla +tags: +- generated_from_trainer +- dpo +- trl +licence: license +--- + +# Model Card for dpo_output_vanilla + +This model is a fine-tuned version of [None](https://huggingface.co/None). +It has been trained using [TRL](https://github.com/huggingface/trl). + +## Quick start + +```python +from transformers import pipeline + +question = "If you had a time machine, but could only go to the past or the future once and never return, which would you choose and why?" +generator = pipeline("text-generation", model="None", device="cuda") +output = generator([{"role": "user", "content": question}], max_new_tokens=128, return_full_text=False)[0] +print(output["generated_text"]) +``` + +## Training procedure + +[Visualize in Weights & Biases](https://wandb.ai/ouhenio/llama32-1b-dpo/runs/8p59m9tb) + + + +This model was trained with DPO, a method introduced in [Direct Preference Optimization: Your Language Model is Secretly a Reward Model](https://huggingface.co/papers/2305.18290). + +### Framework versions + +- TRL: 1.6.0 +- Transformers: 5.12.1 +- Pytorch: 2.9.1+cu128 +- Datasets: 5.0.0 +- Tokenizers: 0.22.2 + +## Citations + +Cite DPO as: + +```bibtex +@inproceedings{rafailov2023direct, + title = {{Direct Preference Optimization: Your Language Model is Secretly a Reward Model}}, + author = {Rafael Rafailov and Archit Sharma and Eric Mitchell and Christopher D. Manning and Stefano Ermon and Chelsea Finn}, + year = 2023, + booktitle = {Advances in Neural Information Processing Systems 36: Annual Conference on Neural Information Processing Systems 2023, NeurIPS 2023, New Orleans, LA, USA, December 10 - 16, 2023}, + url = {http://papers.nips.cc/paper_files/paper/2023/hash/a85b405ed65c6477a4fe8302b5e06ce7-Abstract-Conference.html}, + editor = {Alice Oh and Tristan Naumann and Amir Globerson and Kate Saenko and Moritz Hardt and Sergey Levine}, +} +``` + +Cite TRL as: + +```bibtex +@software{vonwerra2020trl, + title = {{TRL: Transformers Reinforcement Learning}}, + author = {von Werra, Leandro and Belkada, Younes and Tunstall, Lewis and Beeching, Edward and Thrush, Tristan and Lambert, Nathan and Huang, Shengyi and Rasul, Kashif and Gallouédec, Quentin}, + license = {Apache-2.0}, + url = {https://github.com/huggingface/trl}, + year = {2020} +} +``` \ No newline at end of file diff --git a/all_results.json b/all_results.json new file mode 100644 index 0000000..9c90719 --- /dev/null +++ b/all_results.json @@ -0,0 +1,20 @@ +{ + "entropy": 1.178030303030303, + "epoch": 1.0, + "logits/chosen": -0.30206525592803823, + "logits/rejected": -0.0929858402560999, + "logps/chosen": -472.09090909090907, + "logps/rejected": -557.5757575757576, + "mean_token_accuracy": 0.7097648002884605, + "num_tokens": 358277178.0, + "rewards/accuracies": 0.7234848484848485, + "rewards/chosen": -0.24461780894886365, + "rewards/margins": 1.0697687322443181, + "rewards/rejected": -1.3149266098484849, + "total_flos": 4.975817267087409e+18, + "train_loss": 0.527936527665322, + "train_runtime": 22045.0555, + "train_samples": 272898, + "train_samples_per_second": 12.379, + "train_steps_per_second": 0.097 +} \ No newline at end of file diff --git a/chat_template.jinja b/chat_template.jinja new file mode 100644 index 0000000..78d6037 --- /dev/null +++ b/chat_template.jinja @@ -0,0 +1,5 @@ +{% for message in messages %}{% if loop.index0 == 0 %}{{ bos_token }}{% endif %}<|start_header_id|>{{ message['role'] }}<|end_header_id|> + +{% if message['role'] == 'assistant' %}{% generation %}{{ message['content'] | trim }}<|eot_id|>{% endgeneration %}{% else %}{{ message['content'] | trim }}<|eot_id|>{% endif %}{% endfor %}{% if add_generation_prompt %}{{ '<|start_header_id|>assistant<|end_header_id|> + +' }}{% endif %} \ No newline at end of file diff --git a/config.json b/config.json new file mode 100644 index 0000000..3fdb61a --- /dev/null +++ b/config.json @@ -0,0 +1,36 @@ +{ + "architectures": [ + "LlamaForCausalLM" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "bos_token_id": 128000, + "dtype": "bfloat16", + "eos_token_id": 128001, + "head_dim": 64, + "hidden_act": "silu", + "hidden_size": 2048, + "initializer_range": 0.02, + "intermediate_size": 8192, + "max_position_embeddings": 131072, + "mlp_bias": false, + "model_type": "llama", + "num_attention_heads": 32, + "num_hidden_layers": 16, + "num_key_value_heads": 8, + "pad_token_id": 128001, + "pretraining_tp": 1, + "rms_norm_eps": 1e-05, + "rope_parameters": { + "factor": 32.0, + "high_freq_factor": 4.0, + "low_freq_factor": 1.0, + "original_max_position_embeddings": 8192, + "rope_theta": 500000.0, + "rope_type": "llama3" + }, + "tie_word_embeddings": true, + "transformers_version": "5.12.1", + "use_cache": false, + "vocab_size": 128256 +} diff --git a/generation_config.json b/generation_config.json new file mode 100644 index 0000000..962b004 --- /dev/null +++ b/generation_config.json @@ -0,0 +1,12 @@ +{ + "_from_model_config": true, + "bos_token_id": 128000, + "do_sample": true, + "eos_token_id": [ + 128001 + ], + "pad_token_id": 128001, + "temperature": 0.6, + "top_p": 0.9, + "transformers_version": "5.12.1" +} diff --git a/model.safetensors b/model.safetensors new file mode 100644 index 0000000..d2dac88 --- /dev/null +++ b/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fbe99621483fb7061cdfc5d4424fc81f040993e8bb92403e441f714193e30cb7 +size 2996982344 diff --git a/tokenizer.json b/tokenizer.json new file mode 100644 index 0000000..1c1d8d5 --- /dev/null +++ b/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6b9e4e7fb171f92fd137b777cc2714bf87d11576700a1dcd7a399e7bbe39537b +size 17209920 diff --git a/tokenizer_config.json b/tokenizer_config.json new file mode 100644 index 0000000..317f802 --- /dev/null +++ b/tokenizer_config.json @@ -0,0 +1,15 @@ +{ + "backend": "tokenizers", + "bos_token": "<|begin_of_text|>", + "clean_up_tokenization_spaces": true, + "eos_token": "<|end_of_text|>", + "is_local": true, + "local_files_only": false, + "model_input_names": [ + "input_ids", + "attention_mask" + ], + "model_max_length": 131072, + "pad_token": "<|end_of_text|>", + "tokenizer_class": "TokenizersBackend" +} diff --git a/train_results.json b/train_results.json new file mode 100644 index 0000000..9c90719 --- /dev/null +++ b/train_results.json @@ -0,0 +1,20 @@ +{ + "entropy": 1.178030303030303, + "epoch": 1.0, + "logits/chosen": -0.30206525592803823, + "logits/rejected": -0.0929858402560999, + "logps/chosen": -472.09090909090907, + "logps/rejected": -557.5757575757576, + "mean_token_accuracy": 0.7097648002884605, + "num_tokens": 358277178.0, + "rewards/accuracies": 0.7234848484848485, + "rewards/chosen": -0.24461780894886365, + "rewards/margins": 1.0697687322443181, + "rewards/rejected": -1.3149266098484849, + "total_flos": 4.975817267087409e+18, + "train_loss": 0.527936527665322, + "train_runtime": 22045.0555, + "train_samples": 272898, + "train_samples_per_second": 12.379, + "train_steps_per_second": 0.097 +} \ No newline at end of file diff --git a/trainer_state.json b/trainer_state.json new file mode 100644 index 0000000..6663291 --- /dev/null +++ b/trainer_state.json @@ -0,0 +1,3888 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 1.0, + "eval_steps": 500, + "global_step": 2133, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 1.2380126953125, + "epoch": 0.004690294022806555, + "grad_norm": 62.6229133605957, + "learning_rate": 2.102803738317757e-08, + "logits/chosen": 0.13142231863153445, + "logits/rejected": 0.5568989329185723, + "logps/chosen": -488.3, + "logps/rejected": -610.840625, + "loss": 0.6997802734375, + "mean_token_accuracy": 0.7084598530083894, + "num_tokens": 1702319.0, + "rewards/accuracies": 0.27421875, + "rewards/chosen": -0.0020932674407958983, + "rewards/margins": -0.0007374286651611328, + "rewards/rejected": -0.0013402223587036133, + "step": 10 + }, + { + "entropy": 1.2425048828125, + "epoch": 0.00938058804561311, + "grad_norm": 58.14165496826172, + "learning_rate": 4.4392523364485976e-08, + "logits/chosen": 0.21829557273294725, + "logits/rejected": 0.6628808946931506, + "logps/chosen": -464.21875, + "logps/rejected": -625.34375, + "loss": 0.70186767578125, + "mean_token_accuracy": 0.7171314246952534, + "num_tokens": 3396761.0, + "rewards/accuracies": 0.2765625, + "rewards/chosen": 0.0008783265948295593, + "rewards/margins": -0.007377481460571289, + "rewards/rejected": 0.008276915550231934, + "step": 20 + }, + { + "entropy": 1.2296875, + "epoch": 0.014070882068419664, + "grad_norm": 59.69734573364258, + "learning_rate": 6.775700934579438e-08, + "logits/chosen": 0.16846038564981053, + "logits/rejected": 0.5656773570114122, + "logps/chosen": -477.23125, + "logps/rejected": -592.425, + "loss": 0.69720458984375, + "mean_token_accuracy": 0.7160559518262744, + "num_tokens": 5098223.0, + "rewards/accuracies": 0.2640625, + "rewards/chosen": 0.0025088131427764893, + "rewards/margins": 0.0023264408111572264, + "rewards/rejected": 0.00020335018634796143, + "step": 30 + }, + { + "entropy": 1.25751953125, + "epoch": 0.01876117609122622, + "grad_norm": 58.34840774536133, + "learning_rate": 9.11214953271028e-08, + "logits/chosen": 0.19244852821179498, + "logits/rejected": 0.5710635891111505, + "logps/chosen": -456.75625, + "logps/rejected": -600.471875, + "loss": 0.69652099609375, + "mean_token_accuracy": 0.7064009141176939, + "num_tokens": 6711358.0, + "rewards/accuracies": 0.29140625, + "rewards/chosen": 0.0020108461380004884, + "rewards/margins": 0.0012389898300170898, + "rewards/rejected": 0.0007735252380371094, + "step": 40 + }, + { + "entropy": 1.2514892578125, + "epoch": 0.023451470114032775, + "grad_norm": 67.314453125, + "learning_rate": 1.1448598130841122e-07, + "logits/chosen": 0.12212445211997296, + "logits/rejected": 0.4953063663441955, + "logps/chosen": -449.971875, + "logps/rejected": -597.55, + "loss": 0.692919921875, + "mean_token_accuracy": 0.7137674799188971, + "num_tokens": 8355688.0, + "rewards/accuracies": 0.3140625, + "rewards/chosen": 0.0004706621170043945, + "rewards/margins": 0.017674684524536133, + "rewards/rejected": -0.017192131280899046, + "step": 50 + }, + { + "entropy": 1.29541015625, + "epoch": 0.028141764136839328, + "grad_norm": 73.55916595458984, + "learning_rate": 1.378504672897196e-07, + "logits/chosen": 0.09788538624036117, + "logits/rejected": 0.5997201450063375, + "logps/chosen": -483.2125, + "logps/rejected": -697.359375, + "loss": 0.687384033203125, + "mean_token_accuracy": 0.7136357419192791, + "num_tokens": 10102836.0, + "rewards/accuracies": 0.3359375, + "rewards/chosen": 0.0074482947587966915, + "rewards/margins": 0.024856948852539064, + "rewards/rejected": -0.017406320571899413, + "step": 60 + }, + { + "entropy": 1.32314453125, + "epoch": 0.032832058159645884, + "grad_norm": 78.66187286376953, + "learning_rate": 1.61214953271028e-07, + "logits/chosen": 0.1396756814227043, + "logits/rejected": 0.6164610300295253, + "logps/chosen": -519.153125, + "logps/rejected": -679.1875, + "loss": 0.6819580078125, + "mean_token_accuracy": 0.7015179498121142, + "num_tokens": 11789869.0, + "rewards/accuracies": 0.3953125, + "rewards/chosen": 0.00875704288482666, + "rewards/margins": 0.0570986270904541, + "rewards/rejected": -0.04829350709915161, + "step": 70 + }, + { + "entropy": 1.2225830078125, + "epoch": 0.03752235218245244, + "grad_norm": 58.509803771972656, + "learning_rate": 1.8457943925233644e-07, + "logits/chosen": 0.14060797812788714, + "logits/rejected": 0.564689663234594, + "logps/chosen": -405.990625, + "logps/rejected": -560.575, + "loss": 0.67740478515625, + "mean_token_accuracy": 0.7172680467367172, + "num_tokens": 13410425.0, + "rewards/accuracies": 0.42734375, + "rewards/chosen": 0.019669818878173827, + "rewards/margins": 0.06200284957885742, + "rewards/rejected": -0.042318296432495114, + "step": 80 + }, + { + "entropy": 1.2392822265625, + "epoch": 0.04221264620525899, + "grad_norm": 56.662208557128906, + "learning_rate": 2.0794392523364485e-07, + "logits/chosen": 0.06250881791098087, + "logits/rejected": 0.5188753865520929, + "logps/chosen": -479.88125, + "logps/rejected": -573.56875, + "loss": 0.66741943359375, + "mean_token_accuracy": 0.7083912387490272, + "num_tokens": 15075240.0, + "rewards/accuracies": 0.48515625, + "rewards/chosen": 0.018572616577148437, + "rewards/margins": 0.07655124664306641, + "rewards/rejected": -0.058021259307861325, + "step": 90 + }, + { + "entropy": 1.282275390625, + "epoch": 0.04690294022806555, + "grad_norm": 48.26545715332031, + "learning_rate": 2.3130841121495325e-07, + "logits/chosen": 0.01315832664804976, + "logits/rejected": 0.4849252921361867, + "logps/chosen": -465.8, + "logps/rejected": -594.171875, + "loss": 0.65252685546875, + "mean_token_accuracy": 0.7040323404595256, + "num_tokens": 16654416.0, + "rewards/accuracies": 0.51796875, + "rewards/chosen": 0.00697178840637207, + "rewards/margins": 0.1246042251586914, + "rewards/rejected": -0.11760797500610351, + "step": 100 + }, + { + "entropy": 1.2884765625, + "epoch": 0.0515932342508721, + "grad_norm": 47.12166213989258, + "learning_rate": 2.546728971962617e-07, + "logits/chosen": -0.009655450046421743, + "logits/rejected": 0.5041717435221977, + "logps/chosen": -481.08125, + "logps/rejected": -682.14375, + "loss": 0.630169677734375, + "mean_token_accuracy": 0.7061939595267177, + "num_tokens": 18380326.0, + "rewards/accuracies": 0.5640625, + "rewards/chosen": -0.0007643461227416992, + "rewards/margins": 0.24774904251098634, + "rewards/rejected": -0.24857692718505858, + "step": 110 + }, + { + "entropy": 1.245703125, + "epoch": 0.056283528273678655, + "grad_norm": 104.83141326904297, + "learning_rate": 2.780373831775701e-07, + "logits/chosen": 0.07013991718436058, + "logits/rejected": 0.4994907462181204, + "logps/chosen": -465.61875, + "logps/rejected": -618.36875, + "loss": 0.629339599609375, + "mean_token_accuracy": 0.7171272663399577, + "num_tokens": 20071329.0, + "rewards/accuracies": 0.5921875, + "rewards/chosen": -0.009405803680419923, + "rewards/margins": 0.30645027160644533, + "rewards/rejected": -0.3159339904785156, + "step": 120 + }, + { + "entropy": 1.27783203125, + "epoch": 0.06097382229648521, + "grad_norm": 51.31402587890625, + "learning_rate": 3.014018691588785e-07, + "logits/chosen": -0.11757746507376968, + "logits/rejected": 0.3011634539394827, + "logps/chosen": -484.26875, + "logps/rejected": -646.71875, + "loss": 0.61903076171875, + "mean_token_accuracy": 0.7081391334533691, + "num_tokens": 21795525.0, + "rewards/accuracies": 0.59296875, + "rewards/chosen": -0.02664012908935547, + "rewards/margins": 0.32644290924072267, + "rewards/rejected": -0.3529624938964844, + "step": 130 + }, + { + "entropy": 1.2402099609375, + "epoch": 0.06566411631929177, + "grad_norm": 46.31311798095703, + "learning_rate": 3.247663551401869e-07, + "logits/chosen": -0.1286943394512165, + "logits/rejected": 0.2729156901183648, + "logps/chosen": -454.53125, + "logps/rejected": -620.6125, + "loss": 0.60274658203125, + "mean_token_accuracy": 0.7122334752231836, + "num_tokens": 23425866.0, + "rewards/accuracies": 0.6171875, + "rewards/chosen": -0.00044622421264648435, + "rewards/margins": 0.4336221694946289, + "rewards/rejected": -0.43410415649414064, + "step": 140 + }, + { + "entropy": 1.31734619140625, + "epoch": 0.07035441034209831, + "grad_norm": 58.106231689453125, + "learning_rate": 3.481308411214953e-07, + "logits/chosen": -0.1213149723170385, + "logits/rejected": 0.3410684163822822, + "logps/chosen": -504.925, + "logps/rejected": -746.7875, + "loss": 0.6043121337890625, + "mean_token_accuracy": 0.7165910258889199, + "num_tokens": 25164538.0, + "rewards/accuracies": 0.64140625, + "rewards/chosen": -0.05042829513549805, + "rewards/margins": 0.5049018859863281, + "rewards/rejected": -0.5555450439453125, + "step": 150 + }, + { + "entropy": 1.196240234375, + "epoch": 0.07504470436490487, + "grad_norm": 41.2678108215332, + "learning_rate": 3.7149532710280375e-07, + "logits/chosen": -0.20468518942131192, + "logits/rejected": 0.24485665882537527, + "logps/chosen": -453.096875, + "logps/rejected": -595.925, + "loss": 0.5784027099609375, + "mean_token_accuracy": 0.7159464564174414, + "num_tokens": 26805458.0, + "rewards/accuracies": 0.6375, + "rewards/chosen": -0.012469863891601563, + "rewards/margins": 0.47882614135742185, + "rewards/rejected": -0.49114646911621096, + "step": 160 + }, + { + "entropy": 1.3654052734375, + "epoch": 0.07973499838771143, + "grad_norm": 49.10939407348633, + "learning_rate": 3.9485981308411216e-07, + "logits/chosen": -0.35195917230947377, + "logits/rejected": 0.3429667283637607, + "logps/chosen": -479.825, + "logps/rejected": -772.64375, + "loss": 0.6050064086914062, + "mean_token_accuracy": 0.7037448784336447, + "num_tokens": 28484655.0, + "rewards/accuracies": 0.63359375, + "rewards/chosen": -0.06340255737304687, + "rewards/margins": 0.5980648040771485, + "rewards/rejected": -0.6617424011230468, + "step": 170 + }, + { + "entropy": 1.1974609375, + "epoch": 0.08442529241051798, + "grad_norm": 54.43376922607422, + "learning_rate": 4.1822429906542056e-07, + "logits/chosen": -0.25779980917979717, + "logits/rejected": 0.31058484361609595, + "logps/chosen": -471.9875, + "logps/rejected": -559.44375, + "loss": 0.600372314453125, + "mean_token_accuracy": 0.7057331206277013, + "num_tokens": 30109849.0, + "rewards/accuracies": 0.63359375, + "rewards/chosen": -0.08329944610595703, + "rewards/margins": 0.5749111175537109, + "rewards/rejected": -0.6581335067749023, + "step": 180 + }, + { + "entropy": 1.286376953125, + "epoch": 0.08911558643332454, + "grad_norm": 42.45182800292969, + "learning_rate": 4.4158878504672897e-07, + "logits/chosen": -0.35561966840299924, + "logits/rejected": 0.15353543618014337, + "logps/chosen": -463.621875, + "logps/rejected": -656.0375, + "loss": 0.5772369384765625, + "mean_token_accuracy": 0.7057704044505954, + "num_tokens": 31744162.0, + "rewards/accuracies": 0.64921875, + "rewards/chosen": -0.1075469970703125, + "rewards/margins": 0.6416389465332031, + "rewards/rejected": -0.7492580413818359, + "step": 190 + }, + { + "entropy": 1.3052490234375, + "epoch": 0.0938058804561311, + "grad_norm": 62.58420181274414, + "learning_rate": 4.6495327102803737e-07, + "logits/chosen": -0.35697863410661446, + "logits/rejected": 0.20630076585241763, + "logps/chosen": -465.2, + "logps/rejected": -731.7375, + "loss": 0.557763671875, + "mean_token_accuracy": 0.7047198623418808, + "num_tokens": 33449741.0, + "rewards/accuracies": 0.665625, + "rewards/chosen": -0.0918426513671875, + "rewards/margins": 0.718874740600586, + "rewards/rejected": -0.8103523254394531, + "step": 200 + }, + { + "entropy": 1.24949951171875, + "epoch": 0.09849617447893765, + "grad_norm": 38.690673828125, + "learning_rate": 4.883177570093458e-07, + "logits/chosen": -0.36467700401264636, + "logits/rejected": 0.16843314232397516, + "logps/chosen": -500.925, + "logps/rejected": -669.7, + "loss": 0.558367919921875, + "mean_token_accuracy": 0.7056573690846563, + "num_tokens": 35141213.0, + "rewards/accuracies": 0.678125, + "rewards/chosen": -0.13154408931732178, + "rewards/margins": 0.7080429077148438, + "rewards/rejected": -0.8396020889282226, + "step": 210 + }, + { + "entropy": 1.26572265625, + "epoch": 0.1031864685017442, + "grad_norm": 52.72610855102539, + "learning_rate": 4.986972381448671e-07, + "logits/chosen": -0.3342161703831604, + "logits/rejected": 0.13690835608174162, + "logps/chosen": -473.95, + "logps/rejected": -683.790625, + "loss": 0.55330810546875, + "mean_token_accuracy": 0.7077068831771612, + "num_tokens": 36812333.0, + "rewards/accuracies": 0.675, + "rewards/chosen": -0.11907577514648438, + "rewards/margins": 0.7173862457275391, + "rewards/rejected": -0.8365554809570312, + "step": 220 + }, + { + "entropy": 1.18297119140625, + "epoch": 0.10787676252455075, + "grad_norm": 39.5418586730957, + "learning_rate": 4.960917144346014e-07, + "logits/chosen": -0.3157396531611962, + "logits/rejected": 0.06412190059466064, + "logps/chosen": -460.26875, + "logps/rejected": -588.134375, + "loss": 0.55792236328125, + "mean_token_accuracy": 0.7146372459828854, + "num_tokens": 38463618.0, + "rewards/accuracies": 0.67265625, + "rewards/chosen": -0.12630558013916016, + "rewards/margins": 0.7576118469238281, + "rewards/rejected": -0.8838109970092773, + "step": 230 + }, + { + "entropy": 1.2109130859375, + "epoch": 0.11256705654735731, + "grad_norm": 42.412132263183594, + "learning_rate": 4.934861907243356e-07, + "logits/chosen": -0.2700029816915189, + "logits/rejected": 0.16311413886610973, + "logps/chosen": -465.3375, + "logps/rejected": -638.7125, + "loss": 0.5756942749023437, + "mean_token_accuracy": 0.7144404657185077, + "num_tokens": 40153509.0, + "rewards/accuracies": 0.66015625, + "rewards/chosen": -0.17855072021484375, + "rewards/margins": 0.8050395965576171, + "rewards/rejected": -0.983447265625, + "step": 240 + }, + { + "entropy": 1.2478759765625, + "epoch": 0.11725735057016387, + "grad_norm": 52.07088851928711, + "learning_rate": 4.908806670140698e-07, + "logits/chosen": -0.30485234342876655, + "logits/rejected": 0.15554822631006052, + "logps/chosen": -502.325, + "logps/rejected": -663.3875, + "loss": 0.560797119140625, + "mean_token_accuracy": 0.7079653844237328, + "num_tokens": 41888438.0, + "rewards/accuracies": 0.68828125, + "rewards/chosen": -0.08403854370117188, + "rewards/margins": 0.7736038208007813, + "rewards/rejected": -0.8577094078063965, + "step": 250 + }, + { + "entropy": 1.264404296875, + "epoch": 0.12194764459297042, + "grad_norm": 42.2757682800293, + "learning_rate": 4.88275143303804e-07, + "logits/chosen": -0.26204608927871825, + "logits/rejected": 0.16244758132757325, + "logps/chosen": -505.00625, + "logps/rejected": -656.80625, + "loss": 0.577191162109375, + "mean_token_accuracy": 0.7026191920042038, + "num_tokens": 43585062.0, + "rewards/accuracies": 0.66171875, + "rewards/chosen": -0.12207617759704589, + "rewards/margins": 0.7578861236572265, + "rewards/rejected": -0.8799530029296875, + "step": 260 + }, + { + "entropy": 1.2654296875, + "epoch": 0.12663793861577696, + "grad_norm": 37.81559371948242, + "learning_rate": 4.856696195935383e-07, + "logits/chosen": -0.23835901714841606, + "logits/rejected": 0.2375799555447144, + "logps/chosen": -470.54375, + "logps/rejected": -673.821875, + "loss": 0.550146484375, + "mean_token_accuracy": 0.7080739632248878, + "num_tokens": 45300333.0, + "rewards/accuracies": 0.66484375, + "rewards/chosen": -0.06469240188598632, + "rewards/margins": 0.7684661865234375, + "rewards/rejected": -0.8330526351928711, + "step": 270 + }, + { + "entropy": 1.279296875, + "epoch": 0.13132823263858354, + "grad_norm": 90.60016632080078, + "learning_rate": 4.830640958832725e-07, + "logits/chosen": -0.21629677708878328, + "logits/rejected": 0.3163977187624283, + "logps/chosen": -472.6375, + "logps/rejected": -703.2375, + "loss": 0.5496139526367188, + "mean_token_accuracy": 0.7126924116164446, + "num_tokens": 47012468.0, + "rewards/accuracies": 0.675, + "rewards/chosen": -0.1156158447265625, + "rewards/margins": 0.8513694763183594, + "rewards/rejected": -0.9671096801757812, + "step": 280 + }, + { + "entropy": 1.2152099609375, + "epoch": 0.13601852666139008, + "grad_norm": 45.61990737915039, + "learning_rate": 4.804585721730068e-07, + "logits/chosen": -0.22120995301749558, + "logits/rejected": 0.15842283279794905, + "logps/chosen": -464.7625, + "logps/rejected": -603.35625, + "loss": 0.5582733154296875, + "mean_token_accuracy": 0.7125538818538188, + "num_tokens": 48630480.0, + "rewards/accuracies": 0.69140625, + "rewards/chosen": -0.1512277603149414, + "rewards/margins": 0.7555118560791015, + "rewards/rejected": -0.9074989318847656, + "step": 290 + }, + { + "entropy": 1.3152099609375, + "epoch": 0.14070882068419663, + "grad_norm": 48.600868225097656, + "learning_rate": 4.77853048462741e-07, + "logits/chosen": -0.2612698061733851, + "logits/rejected": 0.27788530891313673, + "logps/chosen": -461.784375, + "logps/rejected": -702.3375, + "loss": 0.5443603515625, + "mean_token_accuracy": 0.7086123580113053, + "num_tokens": 50268655.0, + "rewards/accuracies": 0.696875, + "rewards/chosen": -0.11336288452148438, + "rewards/margins": 0.8591796875, + "rewards/rejected": -0.9724113464355468, + "step": 300 + }, + { + "entropy": 1.25264892578125, + "epoch": 0.1453991147070032, + "grad_norm": 55.53269958496094, + "learning_rate": 4.752475247524752e-07, + "logits/chosen": -0.29287300945492223, + "logits/rejected": 0.16384651433880326, + "logps/chosen": -474.85625, + "logps/rejected": -628.31875, + "loss": 0.5508621215820313, + "mean_token_accuracy": 0.7046513512730599, + "num_tokens": 51907656.0, + "rewards/accuracies": 0.67265625, + "rewards/chosen": -0.0640228271484375, + "rewards/margins": 0.7980010986328125, + "rewards/rejected": -0.8622137069702148, + "step": 310 + }, + { + "entropy": 1.22216796875, + "epoch": 0.15008940872980975, + "grad_norm": 42.761417388916016, + "learning_rate": 4.726420010422095e-07, + "logits/chosen": -0.25384103775604866, + "logits/rejected": 0.22921325218368754, + "logps/chosen": -459.175, + "logps/rejected": -641.70625, + "loss": 0.5528564453125, + "mean_token_accuracy": 0.7093667019158602, + "num_tokens": 53593177.0, + "rewards/accuracies": 0.68828125, + "rewards/chosen": -0.13062238693237305, + "rewards/margins": 0.8838020324707031, + "rewards/rejected": -1.0145111083984375, + "step": 320 + }, + { + "entropy": 1.268212890625, + "epoch": 0.1547797027526163, + "grad_norm": 54.23225021362305, + "learning_rate": 4.7003647733194366e-07, + "logits/chosen": -0.2853499345335391, + "logits/rejected": 0.26540551703707715, + "logps/chosen": -485.53125, + "logps/rejected": -700.928125, + "loss": 0.5279586791992188, + "mean_token_accuracy": 0.7084382649511098, + "num_tokens": 55282207.0, + "rewards/accuracies": 0.69609375, + "rewards/chosen": -0.11557159423828126, + "rewards/margins": 0.9410869598388671, + "rewards/rejected": -1.0567031860351563, + "step": 330 + }, + { + "entropy": 1.276025390625, + "epoch": 0.15946999677542287, + "grad_norm": 46.780303955078125, + "learning_rate": 4.6743095362167794e-07, + "logits/chosen": -0.24054302220386106, + "logits/rejected": 0.19597009005313973, + "logps/chosen": -500.8625, + "logps/rejected": -639.490625, + "loss": 0.5339645385742188, + "mean_token_accuracy": 0.7041525453329086, + "num_tokens": 56953401.0, + "rewards/accuracies": 0.70078125, + "rewards/chosen": -0.06949176788330078, + "rewards/margins": 0.908062744140625, + "rewards/rejected": -0.977838134765625, + "step": 340 + }, + { + "entropy": 1.23541259765625, + "epoch": 0.1641602907982294, + "grad_norm": 40.72328567504883, + "learning_rate": 4.6482542991141217e-07, + "logits/chosen": -0.3234164875089639, + "logits/rejected": 0.045452649283336415, + "logps/chosen": -488.621875, + "logps/rejected": -626.525, + "loss": 0.5379608154296875, + "mean_token_accuracy": 0.7138542663305998, + "num_tokens": 58589968.0, + "rewards/accuracies": 0.6859375, + "rewards/chosen": -0.15412139892578125, + "rewards/margins": 0.9932891845703125, + "rewards/rejected": -1.147345733642578, + "step": 350 + }, + { + "entropy": 1.2406005859375, + "epoch": 0.16885058482103596, + "grad_norm": 75.66559600830078, + "learning_rate": 4.6221990620114644e-07, + "logits/chosen": -0.2896619681440237, + "logits/rejected": 0.14065292768913057, + "logps/chosen": -470.790625, + "logps/rejected": -633.634375, + "loss": 0.53887939453125, + "mean_token_accuracy": 0.7099913412705063, + "num_tokens": 60306336.0, + "rewards/accuracies": 0.6796875, + "rewards/chosen": -0.11739025115966797, + "rewards/margins": 0.9522857666015625, + "rewards/rejected": -1.07008056640625, + "step": 360 + }, + { + "entropy": 1.2303955078125, + "epoch": 0.17354087884384253, + "grad_norm": 44.52969741821289, + "learning_rate": 4.596143824908806e-07, + "logits/chosen": -0.3331617374520748, + "logits/rejected": 0.14742571109351904, + "logps/chosen": -439.86875, + "logps/rejected": -650.50625, + "loss": 0.5416946411132812, + "mean_token_accuracy": 0.7148818030953408, + "num_tokens": 61945687.0, + "rewards/accuracies": 0.678125, + "rewards/chosen": -0.0981292724609375, + "rewards/margins": 0.8962696075439454, + "rewards/rejected": -0.9943016052246094, + "step": 370 + }, + { + "entropy": 1.19935302734375, + "epoch": 0.17823117286664908, + "grad_norm": 63.72592544555664, + "learning_rate": 4.570088587806149e-07, + "logits/chosen": -0.3539870810775333, + "logits/rejected": 0.0757170174215348, + "logps/chosen": -486.1875, + "logps/rejected": -613.4, + "loss": 0.5257133483886719, + "mean_token_accuracy": 0.70884978659451, + "num_tokens": 63623916.0, + "rewards/accuracies": 0.6984375, + "rewards/chosen": -0.15736675262451172, + "rewards/margins": 0.9691802978515625, + "rewards/rejected": -1.1265029907226562, + "step": 380 + }, + { + "entropy": 1.2439208984375, + "epoch": 0.18292146688945563, + "grad_norm": 54.821414947509766, + "learning_rate": 4.544033350703491e-07, + "logits/chosen": -0.3400324315735825, + "logits/rejected": 0.1258022766480983, + "logps/chosen": -510.83125, + "logps/rejected": -640.46875, + "loss": 0.5287796020507812, + "mean_token_accuracy": 0.706388033926487, + "num_tokens": 65288521.0, + "rewards/accuracies": 0.6828125, + "rewards/chosen": -0.19127464294433594, + "rewards/margins": 0.9656356811523438, + "rewards/rejected": -1.15693359375, + "step": 390 + }, + { + "entropy": 1.264453125, + "epoch": 0.1876117609122622, + "grad_norm": 41.4052848815918, + "learning_rate": 4.517978113600834e-07, + "logits/chosen": -0.2976419383716833, + "logits/rejected": 0.12311620193998565, + "logps/chosen": -494.665625, + "logps/rejected": -675.7125, + "loss": 0.5210617065429688, + "mean_token_accuracy": 0.7038448538631201, + "num_tokens": 66997155.0, + "rewards/accuracies": 0.69921875, + "rewards/chosen": -0.16325674057006836, + "rewards/margins": 1.0209686279296875, + "rewards/rejected": -1.184283447265625, + "step": 400 + }, + { + "entropy": 1.2718505859375, + "epoch": 0.19230205493506874, + "grad_norm": 45.782466888427734, + "learning_rate": 4.491922876498176e-07, + "logits/chosen": -0.36419587700943057, + "logits/rejected": 0.09847667197489854, + "logps/chosen": -486.46875, + "logps/rejected": -711.55625, + "loss": 0.5182464599609375, + "mean_token_accuracy": 0.7099555041640997, + "num_tokens": 68686843.0, + "rewards/accuracies": 0.69296875, + "rewards/chosen": -0.14240741729736328, + "rewards/margins": 1.0581985473632813, + "rewards/rejected": -1.2014225006103516, + "step": 410 + }, + { + "entropy": 1.33214111328125, + "epoch": 0.1969923489578753, + "grad_norm": 40.87909698486328, + "learning_rate": 4.465867639395518e-07, + "logits/chosen": -0.3484446268901798, + "logits/rejected": 0.2495398141798188, + "logps/chosen": -444.46875, + "logps/rejected": -750.2875, + "loss": 0.528167724609375, + "mean_token_accuracy": 0.7072742033749819, + "num_tokens": 70338632.0, + "rewards/accuracies": 0.71171875, + "rewards/chosen": -0.1199951171875, + "rewards/margins": 1.0843963623046875, + "rewards/rejected": -1.2041954040527343, + "step": 420 + }, + { + "entropy": 1.277685546875, + "epoch": 0.20168264298068186, + "grad_norm": 45.7933235168457, + "learning_rate": 4.439812402292861e-07, + "logits/chosen": -0.32845346277744014, + "logits/rejected": 0.05965557834924514, + "logps/chosen": -453.76875, + "logps/rejected": -691.446875, + "loss": 0.5423171997070313, + "mean_token_accuracy": 0.711910505220294, + "num_tokens": 72020519.0, + "rewards/accuracies": 0.68515625, + "rewards/chosen": -0.13662586212158204, + "rewards/margins": 1.0402114868164063, + "rewards/rejected": -1.1772491455078125, + "step": 430 + }, + { + "entropy": 1.237890625, + "epoch": 0.2063729370034884, + "grad_norm": 49.85696029663086, + "learning_rate": 4.4137571651902026e-07, + "logits/chosen": -0.32513024103298, + "logits/rejected": 0.10268295673836134, + "logps/chosen": -483.075, + "logps/rejected": -637.725, + "loss": 0.5276473999023438, + "mean_token_accuracy": 0.710579814761877, + "num_tokens": 73698147.0, + "rewards/accuracies": 0.70390625, + "rewards/chosen": -0.11184616088867187, + "rewards/margins": 1.006769561767578, + "rewards/rejected": -1.118536376953125, + "step": 440 + }, + { + "entropy": 1.237841796875, + "epoch": 0.21106323102629496, + "grad_norm": 42.590423583984375, + "learning_rate": 4.3877019280875453e-07, + "logits/chosen": -0.35171392704646287, + "logits/rejected": 0.17345263794266536, + "logps/chosen": -464.83125, + "logps/rejected": -657.575, + "loss": 0.5287338256835937, + "mean_token_accuracy": 0.7119293544441462, + "num_tokens": 75384108.0, + "rewards/accuracies": 0.68203125, + "rewards/chosen": -0.10481681823730468, + "rewards/margins": 1.0241561889648438, + "rewards/rejected": -1.1294021606445312, + "step": 450 + }, + { + "entropy": 1.261865234375, + "epoch": 0.2157535250491015, + "grad_norm": 42.76150131225586, + "learning_rate": 4.3616466909848876e-07, + "logits/chosen": -0.18868256814838458, + "logits/rejected": 0.3261802948037552, + "logps/chosen": -476.89375, + "logps/rejected": -704.0625, + "loss": 0.5230316162109375, + "mean_token_accuracy": 0.7178037509322166, + "num_tokens": 77095077.0, + "rewards/accuracies": 0.7046875, + "rewards/chosen": -0.11411991119384765, + "rewards/margins": 1.1168342590332032, + "rewards/rejected": -1.2309730529785157, + "step": 460 + }, + { + "entropy": 1.2946533203125, + "epoch": 0.22044381907190808, + "grad_norm": 63.289634704589844, + "learning_rate": 4.3355914538822304e-07, + "logits/chosen": -0.2706520637717097, + "logits/rejected": 0.2805214967390196, + "logps/chosen": -481.93125, + "logps/rejected": -709.6875, + "loss": 0.5318412780761719, + "mean_token_accuracy": 0.7098235335201025, + "num_tokens": 78734566.0, + "rewards/accuracies": 0.690625, + "rewards/chosen": -0.1305774688720703, + "rewards/margins": 1.0886627197265626, + "rewards/rejected": -1.2193614959716796, + "step": 470 + }, + { + "entropy": 1.1965087890625, + "epoch": 0.22513411309471462, + "grad_norm": 43.6287841796875, + "learning_rate": 4.309536216779572e-07, + "logits/chosen": -0.33620425129897835, + "logits/rejected": 0.16308440620940146, + "logps/chosen": -424.91875, + "logps/rejected": -613.725, + "loss": 0.5122772216796875, + "mean_token_accuracy": 0.7142355434596539, + "num_tokens": 80328183.0, + "rewards/accuracies": 0.7171875, + "rewards/chosen": -0.1288132667541504, + "rewards/margins": 1.0819381713867187, + "rewards/rejected": -1.2110176086425781, + "step": 480 + }, + { + "entropy": 1.2611328125, + "epoch": 0.22982440711752117, + "grad_norm": 67.60916137695312, + "learning_rate": 4.283480979676915e-07, + "logits/chosen": -0.28429071871907674, + "logits/rejected": 0.16387335536423142, + "logps/chosen": -468.709375, + "logps/rejected": -726.628125, + "loss": 0.5209976196289062, + "mean_token_accuracy": 0.7114728279411793, + "num_tokens": 82035476.0, + "rewards/accuracies": 0.6984375, + "rewards/chosen": -0.09270973205566406, + "rewards/margins": 1.1327011108398437, + "rewards/rejected": -1.225337600708008, + "step": 490 + }, + { + "entropy": 1.2105712890625, + "epoch": 0.23451470114032774, + "grad_norm": 45.96240234375, + "learning_rate": 4.257425742574257e-07, + "logits/chosen": -0.2853582343226048, + "logits/rejected": 0.11035681701698016, + "logps/chosen": -459.8875, + "logps/rejected": -625.590625, + "loss": 0.55081787109375, + "mean_token_accuracy": 0.7172987235710024, + "num_tokens": 83724940.0, + "rewards/accuracies": 0.690625, + "rewards/chosen": -0.16839942932128907, + "rewards/margins": 1.0144439697265626, + "rewards/rejected": -1.1823104858398437, + "step": 500 + }, + { + "entropy": 1.2220703125, + "epoch": 0.2392049951631343, + "grad_norm": 291.7288818359375, + "learning_rate": 4.2313705054716e-07, + "logits/chosen": -0.3063585163455549, + "logits/rejected": 0.2143574987077231, + "logps/chosen": -453.5375, + "logps/rejected": -601.79375, + "loss": 0.5269645690917969, + "mean_token_accuracy": 0.7116759425029159, + "num_tokens": 85405053.0, + "rewards/accuracies": 0.68046875, + "rewards/chosen": -0.10627117156982421, + "rewards/margins": 1.0517967224121094, + "rewards/rejected": -1.1577285766601562, + "step": 510 + }, + { + "entropy": 1.186474609375, + "epoch": 0.24389528918594083, + "grad_norm": 49.72892761230469, + "learning_rate": 4.2053152683689417e-07, + "logits/chosen": -0.35408713711909, + "logits/rejected": 0.08242981100965566, + "logps/chosen": -483.99375, + "logps/rejected": -616.715625, + "loss": 0.5304771423339844, + "mean_token_accuracy": 0.7096766533330083, + "num_tokens": 87075810.0, + "rewards/accuracies": 0.7109375, + "rewards/chosen": -0.1855466842651367, + "rewards/margins": 1.030268096923828, + "rewards/rejected": -1.2156890869140624, + "step": 520 + }, + { + "entropy": 1.2286376953125, + "epoch": 0.2485855832087474, + "grad_norm": 41.44746398925781, + "learning_rate": 4.1792600312662845e-07, + "logits/chosen": -0.42685112278879933, + "logits/rejected": 0.028443143035817398, + "logps/chosen": -472.765625, + "logps/rejected": -648.171875, + "loss": 0.5234077453613282, + "mean_token_accuracy": 0.7097115091979503, + "num_tokens": 88775166.0, + "rewards/accuracies": 0.6984375, + "rewards/chosen": -0.17693424224853516, + "rewards/margins": 1.093129539489746, + "rewards/rejected": -1.2699623107910156, + "step": 530 + }, + { + "entropy": 1.24697265625, + "epoch": 0.2532758772315539, + "grad_norm": 67.51264190673828, + "learning_rate": 4.153204794163627e-07, + "logits/chosen": -0.30632683199230437, + "logits/rejected": 0.09340387012174516, + "logps/chosen": -474.659375, + "logps/rejected": -663.43125, + "loss": 0.5192657470703125, + "mean_token_accuracy": 0.7120445661246777, + "num_tokens": 90483291.0, + "rewards/accuracies": 0.7078125, + "rewards/chosen": -0.1671518325805664, + "rewards/margins": 1.0724746704101562, + "rewards/rejected": -1.23973388671875, + "step": 540 + }, + { + "entropy": 1.254345703125, + "epoch": 0.2579661712543605, + "grad_norm": 45.99155807495117, + "learning_rate": 4.1271495570609696e-07, + "logits/chosen": -0.23550229351542282, + "logits/rejected": 0.1451573755983327, + "logps/chosen": -510.16875, + "logps/rejected": -668.446875, + "loss": 0.5291549682617187, + "mean_token_accuracy": 0.7142287530004978, + "num_tokens": 92221920.0, + "rewards/accuracies": 0.68671875, + "rewards/chosen": -0.22687950134277343, + "rewards/margins": 1.0468917846679688, + "rewards/rejected": -1.2740165710449218, + "step": 550 + }, + { + "entropy": 1.2295654296875, + "epoch": 0.2626564652771671, + "grad_norm": 45.227359771728516, + "learning_rate": 4.1010943199583113e-07, + "logits/chosen": -0.28406279958537584, + "logits/rejected": 0.1589292666080559, + "logps/chosen": -460.29375, + "logps/rejected": -637.91875, + "loss": 0.54854736328125, + "mean_token_accuracy": 0.7158264957368374, + "num_tokens": 93930470.0, + "rewards/accuracies": 0.684375, + "rewards/chosen": -0.1995685577392578, + "rewards/margins": 1.00377197265625, + "rewards/rejected": -1.2029953002929688, + "step": 560 + }, + { + "entropy": 1.3091552734375, + "epoch": 0.2673467592999736, + "grad_norm": 41.385459899902344, + "learning_rate": 4.0750390828556536e-07, + "logits/chosen": -0.2780874213954464, + "logits/rejected": 0.19283398683678324, + "logps/chosen": -493.2875, + "logps/rejected": -737.296875, + "loss": 0.514007568359375, + "mean_token_accuracy": 0.7038853816688061, + "num_tokens": 95674576.0, + "rewards/accuracies": 0.70859375, + "rewards/chosen": -0.10278053283691406, + "rewards/margins": 1.14259033203125, + "rewards/rejected": -1.2454254150390625, + "step": 570 + }, + { + "entropy": 1.2052001953125, + "epoch": 0.27203705332278016, + "grad_norm": 40.99026107788086, + "learning_rate": 4.0489838457529964e-07, + "logits/chosen": -0.33785028651093185, + "logits/rejected": 0.1845951028028979, + "logps/chosen": -474.328125, + "logps/rejected": -641.23125, + "loss": 0.5077590942382812, + "mean_token_accuracy": 0.7113756321370601, + "num_tokens": 97360867.0, + "rewards/accuracies": 0.71796875, + "rewards/chosen": -0.12218704223632812, + "rewards/margins": 1.0552566528320313, + "rewards/rejected": -1.17777099609375, + "step": 580 + }, + { + "entropy": 1.2611083984375, + "epoch": 0.27672734734558674, + "grad_norm": 49.85224151611328, + "learning_rate": 4.022928608650338e-07, + "logits/chosen": -0.3494676075196345, + "logits/rejected": 0.17322246770157881, + "logps/chosen": -486.14375, + "logps/rejected": -657.125, + "loss": 0.53740234375, + "mean_token_accuracy": 0.70248509272933, + "num_tokens": 99032070.0, + "rewards/accuracies": 0.6875, + "rewards/chosen": -0.19557952880859375, + "rewards/margins": 1.0435150146484375, + "rewards/rejected": -1.2389533996582032, + "step": 590 + }, + { + "entropy": 1.219482421875, + "epoch": 0.28141764136839326, + "grad_norm": 44.720306396484375, + "learning_rate": 3.996873371547681e-07, + "logits/chosen": -0.3348284376731146, + "logits/rejected": 0.042711471660991984, + "logps/chosen": -475.38125, + "logps/rejected": -685.36875, + "loss": 0.5462005615234375, + "mean_token_accuracy": 0.7143606793135404, + "num_tokens": 100746662.0, + "rewards/accuracies": 0.69296875, + "rewards/chosen": -0.20562248229980468, + "rewards/margins": 1.092462921142578, + "rewards/rejected": -1.2979248046875, + "step": 600 + }, + { + "entropy": 1.28480224609375, + "epoch": 0.28610793539119983, + "grad_norm": 42.72548294067383, + "learning_rate": 3.970818134445023e-07, + "logits/chosen": -0.39000245026929997, + "logits/rejected": 0.23665966765112026, + "logps/chosen": -449.0, + "logps/rejected": -775.440625, + "loss": 0.47276611328125, + "mean_token_accuracy": 0.71767991296947, + "num_tokens": 102474099.0, + "rewards/accuracies": 0.725, + "rewards/chosen": -0.17837677001953126, + "rewards/margins": 1.2704742431640625, + "rewards/rejected": -1.4485992431640624, + "step": 610 + }, + { + "entropy": 1.24482421875, + "epoch": 0.2907982294140064, + "grad_norm": 58.969539642333984, + "learning_rate": 3.944762897342366e-07, + "logits/chosen": -0.27561446925400485, + "logits/rejected": 0.08348458751869889, + "logps/chosen": -479.00625, + "logps/rejected": -641.2, + "loss": 0.5534393310546875, + "mean_token_accuracy": 0.7048881415277719, + "num_tokens": 104130552.0, + "rewards/accuracies": 0.70234375, + "rewards/chosen": -0.22642822265625, + "rewards/margins": 1.002734375, + "rewards/rejected": -1.2297664642333985, + "step": 620 + }, + { + "entropy": 1.201513671875, + "epoch": 0.2954885234368129, + "grad_norm": 39.077484130859375, + "learning_rate": 3.9187076602397077e-07, + "logits/chosen": -0.301554321263415, + "logits/rejected": 0.21063951345062448, + "logps/chosen": -446.575, + "logps/rejected": -647.35625, + "loss": 0.49993896484375, + "mean_token_accuracy": 0.7235580589622259, + "num_tokens": 105816792.0, + "rewards/accuracies": 0.721875, + "rewards/chosen": -0.1338104248046875, + "rewards/margins": 1.1701812744140625, + "rewards/rejected": -1.3043838500976563, + "step": 630 + }, + { + "entropy": 1.2219970703125, + "epoch": 0.3001788174596195, + "grad_norm": 67.47047424316406, + "learning_rate": 3.8926524231370505e-07, + "logits/chosen": -0.2605092185300394, + "logits/rejected": 0.03782858556336989, + "logps/chosen": -487.1375, + "logps/rejected": -570.540625, + "loss": 0.5586784362792969, + "mean_token_accuracy": 0.7060683678835631, + "num_tokens": 107434278.0, + "rewards/accuracies": 0.684375, + "rewards/chosen": -0.20991897583007812, + "rewards/margins": 0.9743408203125, + "rewards/rejected": -1.1841949462890624, + "step": 640 + }, + { + "entropy": 1.23212890625, + "epoch": 0.30486911148242607, + "grad_norm": 43.978355407714844, + "learning_rate": 3.866597186034393e-07, + "logits/chosen": -0.3408145959077753, + "logits/rejected": 0.05489902388052849, + "logps/chosen": -469.74375, + "logps/rejected": -648.896875, + "loss": 0.530322265625, + "mean_token_accuracy": 0.7166027266532182, + "num_tokens": 109108061.0, + "rewards/accuracies": 0.7, + "rewards/chosen": -0.18619918823242188, + "rewards/margins": 1.049748992919922, + "rewards/rejected": -1.2358810424804687, + "step": 650 + }, + { + "entropy": 1.2399169921875, + "epoch": 0.3095594055052326, + "grad_norm": 43.92350387573242, + "learning_rate": 3.8405419489317355e-07, + "logits/chosen": -0.3518982698881636, + "logits/rejected": 0.15893815329437216, + "logps/chosen": -458.603125, + "logps/rejected": -696.175, + "loss": 0.5294532775878906, + "mean_token_accuracy": 0.7165129277855158, + "num_tokens": 110839489.0, + "rewards/accuracies": 0.7078125, + "rewards/chosen": -0.16309404373168945, + "rewards/margins": 1.1441436767578126, + "rewards/rejected": -1.30704345703125, + "step": 660 + }, + { + "entropy": 1.2388427734375, + "epoch": 0.31424969952803916, + "grad_norm": 45.37200164794922, + "learning_rate": 3.8144867118290773e-07, + "logits/chosen": -0.36028611481310185, + "logits/rejected": 0.1766612002287472, + "logps/chosen": -477.196875, + "logps/rejected": -692.225, + "loss": 0.48761749267578125, + "mean_token_accuracy": 0.7143629230558872, + "num_tokens": 112533599.0, + "rewards/accuracies": 0.7140625, + "rewards/chosen": -0.15464000701904296, + "rewards/margins": 1.1968528747558593, + "rewards/rejected": -1.350921630859375, + "step": 670 + }, + { + "entropy": 1.2490478515625, + "epoch": 0.31893999355084574, + "grad_norm": 50.02637481689453, + "learning_rate": 3.78843147472642e-07, + "logits/chosen": -0.43763616453963217, + "logits/rejected": 0.03516448731243771, + "logps/chosen": -473.65, + "logps/rejected": -714.50625, + "loss": 0.5369621276855469, + "mean_token_accuracy": 0.711085844412446, + "num_tokens": 114210162.0, + "rewards/accuracies": 0.675, + "rewards/chosen": -0.20833778381347656, + "rewards/margins": 1.1489297866821289, + "rewards/rejected": -1.357391357421875, + "step": 680 + }, + { + "entropy": 1.2511962890625, + "epoch": 0.32363028757365225, + "grad_norm": 43.783878326416016, + "learning_rate": 3.7623762376237623e-07, + "logits/chosen": -0.4307592521929092, + "logits/rejected": 0.07158832410380966, + "logps/chosen": -449.75, + "logps/rejected": -669.51875, + "loss": 0.5142929077148437, + "mean_token_accuracy": 0.709177390858531, + "num_tokens": 115807028.0, + "rewards/accuracies": 0.709375, + "rewards/chosen": -0.2126626491546631, + "rewards/margins": 1.17650146484375, + "rewards/rejected": -1.389113998413086, + "step": 690 + }, + { + "entropy": 1.275048828125, + "epoch": 0.3283205815964588, + "grad_norm": 38.663330078125, + "learning_rate": 3.7363210005211046e-07, + "logits/chosen": -0.3787208537319539, + "logits/rejected": 0.12219470055302668, + "logps/chosen": -470.571875, + "logps/rejected": -723.1125, + "loss": 0.5110260009765625, + "mean_token_accuracy": 0.711947762966156, + "num_tokens": 117509330.0, + "rewards/accuracies": 0.71796875, + "rewards/chosen": -0.15255508422851563, + "rewards/margins": 1.1452316284179687, + "rewards/rejected": -1.2983154296875, + "step": 700 + }, + { + "entropy": 1.290673828125, + "epoch": 0.3330108756192654, + "grad_norm": 50.86594772338867, + "learning_rate": 3.710265763418447e-07, + "logits/chosen": -0.36973986713310913, + "logits/rejected": 0.15082829922912258, + "logps/chosen": -490.76875, + "logps/rejected": -683.375, + "loss": 0.5232513427734375, + "mean_token_accuracy": 0.7003827568143606, + "num_tokens": 119183888.0, + "rewards/accuracies": 0.703125, + "rewards/chosen": -0.2656074523925781, + "rewards/margins": 1.10184326171875, + "rewards/rejected": -1.3671539306640625, + "step": 710 + }, + { + "entropy": 1.2464599609375, + "epoch": 0.3377011696420719, + "grad_norm": 77.96504974365234, + "learning_rate": 3.684210526315789e-07, + "logits/chosen": -0.31842311219187563, + "logits/rejected": 0.1074574384256495, + "logps/chosen": -481.25625, + "logps/rejected": -687.425, + "loss": 0.5421051025390625, + "mean_token_accuracy": 0.7150287635624408, + "num_tokens": 120851520.0, + "rewards/accuracies": 0.68203125, + "rewards/chosen": -0.22847366333007812, + "rewards/margins": 1.0441268920898437, + "rewards/rejected": -1.2725669860839843, + "step": 720 + }, + { + "entropy": 1.2686279296875, + "epoch": 0.3423914636648785, + "grad_norm": 47.35414505004883, + "learning_rate": 3.658155289213132e-07, + "logits/chosen": -0.3355926860262508, + "logits/rejected": 0.18558617957879314, + "logps/chosen": -471.8875, + "logps/rejected": -713.5375, + "loss": 0.5218536376953125, + "mean_token_accuracy": 0.7111153559759259, + "num_tokens": 122550214.0, + "rewards/accuracies": 0.70234375, + "rewards/chosen": -0.22301788330078126, + "rewards/margins": 1.0697059631347656, + "rewards/rejected": -1.2926933288574218, + "step": 730 + }, + { + "entropy": 1.28505859375, + "epoch": 0.34708175768768507, + "grad_norm": 49.15150451660156, + "learning_rate": 3.6321000521104736e-07, + "logits/chosen": -0.3326699481098578, + "logits/rejected": 0.01767519845961821, + "logps/chosen": -478.29375, + "logps/rejected": -686.0125, + "loss": 0.560009765625, + "mean_token_accuracy": 0.7098246956244111, + "num_tokens": 124254418.0, + "rewards/accuracies": 0.678125, + "rewards/chosen": -0.24724063873291016, + "rewards/margins": 1.1002532958984375, + "rewards/rejected": -1.3478904724121095, + "step": 740 + }, + { + "entropy": 1.2275146484375, + "epoch": 0.3517720517104916, + "grad_norm": 64.82447814941406, + "learning_rate": 3.6060448150078164e-07, + "logits/chosen": -0.3452819294543103, + "logits/rejected": 0.07315509678314808, + "logps/chosen": -433.484375, + "logps/rejected": -634.0625, + "loss": 0.5266708374023438, + "mean_token_accuracy": 0.7124956920742989, + "num_tokens": 125879625.0, + "rewards/accuracies": 0.69921875, + "rewards/chosen": -0.21740396022796632, + "rewards/margins": 1.1657791137695312, + "rewards/rejected": -1.3828399658203125, + "step": 750 + }, + { + "entropy": 1.2758056640625, + "epoch": 0.35646234573329816, + "grad_norm": 44.01753616333008, + "learning_rate": 3.5799895779051587e-07, + "logits/chosen": -0.3970313479144509, + "logits/rejected": -0.013435611206957002, + "logps/chosen": -483.08125, + "logps/rejected": -656.5125, + "loss": 0.5375274658203125, + "mean_token_accuracy": 0.6993119914084673, + "num_tokens": 127473944.0, + "rewards/accuracies": 0.68671875, + "rewards/chosen": -0.2423839569091797, + "rewards/margins": 1.0922943115234376, + "rewards/rejected": -1.3347213745117188, + "step": 760 + }, + { + "entropy": 1.2775634765625, + "epoch": 0.36115263975610473, + "grad_norm": 49.41606140136719, + "learning_rate": 3.5539343408025015e-07, + "logits/chosen": -0.38184818044372293, + "logits/rejected": 0.1429431066083195, + "logps/chosen": -501.05625, + "logps/rejected": -693.2625, + "loss": 0.5124237060546875, + "mean_token_accuracy": 0.7008192248642444, + "num_tokens": 129129963.0, + "rewards/accuracies": 0.71171875, + "rewards/chosen": -0.22407178878784179, + "rewards/margins": 1.2268234252929688, + "rewards/rejected": -1.450592041015625, + "step": 770 + }, + { + "entropy": 1.280322265625, + "epoch": 0.36584293377891125, + "grad_norm": 46.2809944152832, + "learning_rate": 3.527879103699843e-07, + "logits/chosen": -0.25291314200945403, + "logits/rejected": 0.14969483951627313, + "logps/chosen": -501.875, + "logps/rejected": -715.396875, + "loss": 0.5315521240234375, + "mean_token_accuracy": 0.7103803750127554, + "num_tokens": 130895409.0, + "rewards/accuracies": 0.70078125, + "rewards/chosen": -0.20745162963867186, + "rewards/margins": 1.1037071228027344, + "rewards/rejected": -1.310882568359375, + "step": 780 + }, + { + "entropy": 1.2568115234375, + "epoch": 0.3705332278017178, + "grad_norm": 38.061893463134766, + "learning_rate": 3.501823866597186e-07, + "logits/chosen": -0.29507836911014856, + "logits/rejected": 0.09863675737702185, + "logps/chosen": -479.525, + "logps/rejected": -677.296875, + "loss": 0.5366546630859375, + "mean_token_accuracy": 0.713270222209394, + "num_tokens": 132594219.0, + "rewards/accuracies": 0.68125, + "rewards/chosen": -0.21154518127441407, + "rewards/margins": 1.1149787902832031, + "rewards/rejected": -1.3264785766601563, + "step": 790 + }, + { + "entropy": 1.26630859375, + "epoch": 0.3752235218245244, + "grad_norm": 40.844886779785156, + "learning_rate": 3.4757686294945283e-07, + "logits/chosen": -0.32828585106273067, + "logits/rejected": 0.13410074141871198, + "logps/chosen": -486.453125, + "logps/rejected": -650.13125, + "loss": 0.5410888671875, + "mean_token_accuracy": 0.7061129916459322, + "num_tokens": 134231292.0, + "rewards/accuracies": 0.7046875, + "rewards/chosen": -0.21239662170410156, + "rewards/margins": 1.0793701171875, + "rewards/rejected": -1.2920806884765625, + "step": 800 + }, + { + "entropy": 1.1828125, + "epoch": 0.3799138158473309, + "grad_norm": 36.530025482177734, + "learning_rate": 3.449713392391871e-07, + "logits/chosen": -0.2730355243405189, + "logits/rejected": 0.09156594890767208, + "logps/chosen": -460.49375, + "logps/rejected": -573.3875, + "loss": 0.5164207458496094, + "mean_token_accuracy": 0.7155072972178459, + "num_tokens": 135893775.0, + "rewards/accuracies": 0.6984375, + "rewards/chosen": -0.12217826843261718, + "rewards/margins": 0.9473831176757812, + "rewards/rejected": -1.0695075988769531, + "step": 810 + }, + { + "entropy": 1.22552490234375, + "epoch": 0.3846041098701375, + "grad_norm": 40.324920654296875, + "learning_rate": 3.423658155289213e-07, + "logits/chosen": -0.2511466705433654, + "logits/rejected": 0.22135653984727344, + "logps/chosen": -459.48125, + "logps/rejected": -658.15, + "loss": 0.54027099609375, + "mean_token_accuracy": 0.7177877865731717, + "num_tokens": 137573025.0, + "rewards/accuracies": 0.6875, + "rewards/chosen": -0.17630014419555665, + "rewards/margins": 1.01343994140625, + "rewards/rejected": -1.1897003173828125, + "step": 820 + }, + { + "entropy": 1.2394775390625, + "epoch": 0.38929440389294406, + "grad_norm": 44.12002182006836, + "learning_rate": 3.397602918186555e-07, + "logits/chosen": -0.28912138485015315, + "logits/rejected": 0.19754984016879085, + "logps/chosen": -475.20625, + "logps/rejected": -668.40625, + "loss": 0.5154266357421875, + "mean_token_accuracy": 0.7126389849931002, + "num_tokens": 139224712.0, + "rewards/accuracies": 0.71171875, + "rewards/chosen": -0.1871318817138672, + "rewards/margins": 1.09332275390625, + "rewards/rejected": -1.2801509857177735, + "step": 830 + }, + { + "entropy": 1.246630859375, + "epoch": 0.3939846979157506, + "grad_norm": 87.65414428710938, + "learning_rate": 3.371547681083898e-07, + "logits/chosen": -0.2749478759605691, + "logits/rejected": 0.17435728309129989, + "logps/chosen": -494.425, + "logps/rejected": -700.06875, + "loss": 0.5238235473632813, + "mean_token_accuracy": 0.7123918626457453, + "num_tokens": 140969200.0, + "rewards/accuracies": 0.71796875, + "rewards/chosen": -0.21366329193115235, + "rewards/margins": 1.1942451477050782, + "rewards/rejected": -1.407720947265625, + "step": 840 + }, + { + "entropy": 1.230517578125, + "epoch": 0.39867499193855716, + "grad_norm": 81.68108367919922, + "learning_rate": 3.34549244398124e-07, + "logits/chosen": -0.3473418693014164, + "logits/rejected": 0.0888236907473366, + "logps/chosen": -476.23125, + "logps/rejected": -655.90625, + "loss": 0.5314712524414062, + "mean_token_accuracy": 0.7124969633296132, + "num_tokens": 142707119.0, + "rewards/accuracies": 0.7, + "rewards/chosen": -0.2162473678588867, + "rewards/margins": 1.093731689453125, + "rewards/rejected": -1.3099411010742188, + "step": 850 + }, + { + "entropy": 1.322705078125, + "epoch": 0.40336528596136373, + "grad_norm": 59.91925048828125, + "learning_rate": 3.3194372068785824e-07, + "logits/chosen": -0.3529681888952906, + "logits/rejected": 0.23290446857518904, + "logps/chosen": -443.8625, + "logps/rejected": -743.39375, + "loss": 0.5114044189453125, + "mean_token_accuracy": 0.7079902265220881, + "num_tokens": 144391043.0, + "rewards/accuracies": 0.7296875, + "rewards/chosen": -0.2095262050628662, + "rewards/margins": 1.2830490112304687, + "rewards/rejected": -1.492266845703125, + "step": 860 + }, + { + "entropy": 1.27998046875, + "epoch": 0.40805557998417025, + "grad_norm": 40.34415054321289, + "learning_rate": 3.2933819697759247e-07, + "logits/chosen": -0.38396973862868355, + "logits/rejected": 0.1605909461102152, + "logps/chosen": -454.6, + "logps/rejected": -662.209375, + "loss": 0.52490234375, + "mean_token_accuracy": 0.7066729519516229, + "num_tokens": 145997144.0, + "rewards/accuracies": 0.71484375, + "rewards/chosen": -0.1993091583251953, + "rewards/margins": 1.155511474609375, + "rewards/rejected": -1.3549407958984374, + "step": 870 + }, + { + "entropy": 1.243115234375, + "epoch": 0.4127458740069768, + "grad_norm": 43.68743133544922, + "learning_rate": 3.2673267326732674e-07, + "logits/chosen": -0.3361740698074229, + "logits/rejected": 0.07762473854104804, + "logps/chosen": -509.29375, + "logps/rejected": -656.60625, + "loss": 0.4995269775390625, + "mean_token_accuracy": 0.7099218437448144, + "num_tokens": 147695183.0, + "rewards/accuracies": 0.70859375, + "rewards/chosen": -0.23754005432128905, + "rewards/margins": 1.1845123291015625, + "rewards/rejected": -1.4216781616210938, + "step": 880 + }, + { + "entropy": 1.23310546875, + "epoch": 0.4174361680297834, + "grad_norm": 80.83831024169922, + "learning_rate": 3.241271495570609e-07, + "logits/chosen": -0.3442709798320729, + "logits/rejected": 0.10712556906402045, + "logps/chosen": -499.9625, + "logps/rejected": -665.31875, + "loss": 0.5188461303710937, + "mean_token_accuracy": 0.707976047694683, + "num_tokens": 149444689.0, + "rewards/accuracies": 0.7109375, + "rewards/chosen": -0.2080381393432617, + "rewards/margins": 1.1563003540039063, + "rewards/rejected": -1.3644058227539062, + "step": 890 + }, + { + "entropy": 1.278857421875, + "epoch": 0.4221264620525899, + "grad_norm": 46.61259460449219, + "learning_rate": 3.215216258467952e-07, + "logits/chosen": -0.3779255074786172, + "logits/rejected": 0.13898465945471042, + "logps/chosen": -463.884375, + "logps/rejected": -697.05625, + "loss": 0.509967041015625, + "mean_token_accuracy": 0.7144387796521187, + "num_tokens": 151137840.0, + "rewards/accuracies": 0.72734375, + "rewards/chosen": -0.22325286865234376, + "rewards/margins": 1.138238525390625, + "rewards/rejected": -1.3617431640625, + "step": 900 + }, + { + "entropy": 1.2478271484375, + "epoch": 0.4268167560753965, + "grad_norm": 57.145328521728516, + "learning_rate": 3.189161021365294e-07, + "logits/chosen": -0.4029233645848759, + "logits/rejected": 0.07330459311441123, + "logps/chosen": -447.553125, + "logps/rejected": -658.375, + "loss": 0.536572265625, + "mean_token_accuracy": 0.7051212705671788, + "num_tokens": 152711567.0, + "rewards/accuracies": 0.70546875, + "rewards/chosen": -0.22397823333740235, + "rewards/margins": 1.013479232788086, + "rewards/rejected": -1.2374153137207031, + "step": 910 + }, + { + "entropy": 1.212109375, + "epoch": 0.431507050098203, + "grad_norm": 45.5133171081543, + "learning_rate": 3.163105784262637e-07, + "logits/chosen": -0.3665716216140561, + "logits/rejected": 0.03931592270211523, + "logps/chosen": -450.85625, + "logps/rejected": -642.99375, + "loss": 0.5301528930664062, + "mean_token_accuracy": 0.7148423504084349, + "num_tokens": 154353646.0, + "rewards/accuracies": 0.69140625, + "rewards/chosen": -0.18856353759765626, + "rewards/margins": 1.0661237716674805, + "rewards/rejected": -1.2548416137695313, + "step": 920 + }, + { + "entropy": 1.214599609375, + "epoch": 0.4361973441210096, + "grad_norm": 43.24400329589844, + "learning_rate": 3.137050547159979e-07, + "logits/chosen": -0.30179459918123197, + "logits/rejected": 0.09087620740651976, + "logps/chosen": -494.99375, + "logps/rejected": -610.909375, + "loss": 0.5239151000976563, + "mean_token_accuracy": 0.7096556110307575, + "num_tokens": 156025668.0, + "rewards/accuracies": 0.70703125, + "rewards/chosen": -0.2380859375, + "rewards/margins": 1.0573944091796874, + "rewards/rejected": -1.295379638671875, + "step": 930 + }, + { + "entropy": 1.207666015625, + "epoch": 0.44088763814381615, + "grad_norm": 72.49191284179688, + "learning_rate": 3.1109953100573216e-07, + "logits/chosen": -0.29770645267053186, + "logits/rejected": 0.15372499151275804, + "logps/chosen": -484.06875, + "logps/rejected": -627.015625, + "loss": 0.5368232727050781, + "mean_token_accuracy": 0.7107538601383567, + "num_tokens": 157761123.0, + "rewards/accuracies": 0.703125, + "rewards/chosen": -0.27497024536132814, + "rewards/margins": 1.0991371154785157, + "rewards/rejected": -1.3735931396484375, + "step": 940 + }, + { + "entropy": 1.22421875, + "epoch": 0.44557793216662267, + "grad_norm": 44.017452239990234, + "learning_rate": 3.084940072954664e-07, + "logits/chosen": -0.4168874535254143, + "logits/rejected": 0.011802939570460098, + "logps/chosen": -430.26875, + "logps/rejected": -586.91875, + "loss": 0.5136375427246094, + "mean_token_accuracy": 0.7067252658307552, + "num_tokens": 159297345.0, + "rewards/accuracies": 0.7140625, + "rewards/chosen": -0.20767440795898437, + "rewards/margins": 1.0971954345703125, + "rewards/rejected": -1.30452880859375, + "step": 950 + }, + { + "entropy": 1.25185546875, + "epoch": 0.45026822618942924, + "grad_norm": 66.41146850585938, + "learning_rate": 3.058884835852006e-07, + "logits/chosen": -0.3064041353585244, + "logits/rejected": 0.22961672692960927, + "logps/chosen": -475.13125, + "logps/rejected": -759.978125, + "loss": 0.5192276000976562, + "mean_token_accuracy": 0.7137241207063199, + "num_tokens": 161048146.0, + "rewards/accuracies": 0.71171875, + "rewards/chosen": -0.2347585678100586, + "rewards/margins": 1.1838924407958984, + "rewards/rejected": -1.4190101623535156, + "step": 960 + }, + { + "entropy": 1.2472900390625, + "epoch": 0.4549585202122358, + "grad_norm": 56.4462776184082, + "learning_rate": 3.0328295987493483e-07, + "logits/chosen": -0.3383007687161007, + "logits/rejected": 0.13007250379762667, + "logps/chosen": -473.79375, + "logps/rejected": -676.871875, + "loss": 0.5186408996582031, + "mean_token_accuracy": 0.7102607268840074, + "num_tokens": 162759295.0, + "rewards/accuracies": 0.6984375, + "rewards/chosen": -0.22657947540283202, + "rewards/margins": 1.1189689636230469, + "rewards/rejected": -1.3456756591796875, + "step": 970 + }, + { + "entropy": 1.30703125, + "epoch": 0.45964881423504234, + "grad_norm": 50.83198165893555, + "learning_rate": 3.0067743616466906e-07, + "logits/chosen": -0.2545741993101498, + "logits/rejected": 0.15262439804664002, + "logps/chosen": -483.86875, + "logps/rejected": -712.11875, + "loss": 0.5216323852539062, + "mean_token_accuracy": 0.7064540114253759, + "num_tokens": 164446081.0, + "rewards/accuracies": 0.7078125, + "rewards/chosen": -0.20740795135498047, + "rewards/margins": 1.1302799224853515, + "rewards/rejected": -1.337384033203125, + "step": 980 + }, + { + "entropy": 1.2359375, + "epoch": 0.4643391082578489, + "grad_norm": 55.23470687866211, + "learning_rate": 2.9807191245440334e-07, + "logits/chosen": -0.30537551995678974, + "logits/rejected": 0.11711747734812943, + "logps/chosen": -465.8, + "logps/rejected": -614.971875, + "loss": 0.5563156127929687, + "mean_token_accuracy": 0.7104039791971445, + "num_tokens": 166020948.0, + "rewards/accuracies": 0.67109375, + "rewards/chosen": -0.21481266021728515, + "rewards/margins": 0.9331451416015625, + "rewards/rejected": -1.1478729248046875, + "step": 990 + }, + { + "entropy": 1.2370849609375, + "epoch": 0.4690294022806555, + "grad_norm": 42.55168914794922, + "learning_rate": 2.954663887441375e-07, + "logits/chosen": -0.26711619092213956, + "logits/rejected": 0.21803520041534127, + "logps/chosen": -469.85, + "logps/rejected": -661.95625, + "loss": 0.506292724609375, + "mean_token_accuracy": 0.71516153793782, + "num_tokens": 167714489.0, + "rewards/accuracies": 0.72265625, + "rewards/chosen": -0.22079696655273437, + "rewards/margins": 1.1295989990234374, + "rewards/rejected": -1.3502197265625, + "step": 1000 + }, + { + "entropy": 1.26083984375, + "epoch": 0.473719696303462, + "grad_norm": 57.51393508911133, + "learning_rate": 2.928608650338718e-07, + "logits/chosen": -0.3097135444020215, + "logits/rejected": 0.0631521841392775, + "logps/chosen": -485.55625, + "logps/rejected": -617.9875, + "loss": 0.5513885498046875, + "mean_token_accuracy": 0.7011837583035231, + "num_tokens": 169396307.0, + "rewards/accuracies": 0.67265625, + "rewards/chosen": -0.2332773208618164, + "rewards/margins": 0.9677375793457031, + "rewards/rejected": -1.2006851196289063, + "step": 1010 + }, + { + "entropy": 1.258544921875, + "epoch": 0.4784099903262686, + "grad_norm": 47.03550720214844, + "learning_rate": 2.90255341323606e-07, + "logits/chosen": -0.3139030744878589, + "logits/rejected": 0.1747507997422271, + "logps/chosen": -476.775, + "logps/rejected": -688.54375, + "loss": 0.5033882141113282, + "mean_token_accuracy": 0.7097974337637425, + "num_tokens": 171061030.0, + "rewards/accuracies": 0.72578125, + "rewards/chosen": -0.20766181945800782, + "rewards/margins": 1.085690689086914, + "rewards/rejected": -1.2930282592773437, + "step": 1020 + }, + { + "entropy": 1.2618408203125, + "epoch": 0.48310028434907515, + "grad_norm": 37.497474670410156, + "learning_rate": 2.876498176133403e-07, + "logits/chosen": -0.35845184599210705, + "logits/rejected": 0.06585844424002088, + "logps/chosen": -487.5375, + "logps/rejected": -678.8, + "loss": 0.4933441162109375, + "mean_token_accuracy": 0.7051137916743755, + "num_tokens": 172742451.0, + "rewards/accuracies": 0.72578125, + "rewards/chosen": -0.22617607116699218, + "rewards/margins": 1.1918342590332032, + "rewards/rejected": -1.4184478759765624, + "step": 1030 + }, + { + "entropy": 1.2374267578125, + "epoch": 0.48779057837188167, + "grad_norm": 43.61983871459961, + "learning_rate": 2.8504429390307447e-07, + "logits/chosen": -0.2910774125262612, + "logits/rejected": 0.15164186582642813, + "logps/chosen": -468.56875, + "logps/rejected": -712.00625, + "loss": 0.5211807250976562, + "mean_token_accuracy": 0.7186769794672727, + "num_tokens": 174461002.0, + "rewards/accuracies": 0.70390625, + "rewards/chosen": -0.23565731048583985, + "rewards/margins": 1.1290267944335937, + "rewards/rejected": -1.3653778076171874, + "step": 1040 + }, + { + "entropy": 1.237646484375, + "epoch": 0.49248087239468824, + "grad_norm": 41.82542419433594, + "learning_rate": 2.8243877019280875e-07, + "logits/chosen": -0.27986107767208945, + "logits/rejected": 0.09014670592249303, + "logps/chosen": -482.040625, + "logps/rejected": -618.11875, + "loss": 0.5121337890625, + "mean_token_accuracy": 0.7039212234318256, + "num_tokens": 176123752.0, + "rewards/accuracies": 0.7125, + "rewards/chosen": -0.23637866973876953, + "rewards/margins": 1.126165008544922, + "rewards/rejected": -1.3623748779296876, + "step": 1050 + }, + { + "entropy": 1.244970703125, + "epoch": 0.4971711664174948, + "grad_norm": 39.264984130859375, + "learning_rate": 2.79833246482543e-07, + "logits/chosen": -0.3118509453919952, + "logits/rejected": 0.171299993947704, + "logps/chosen": -454.53125, + "logps/rejected": -675.275, + "loss": 0.5181777954101563, + "mean_token_accuracy": 0.7140839375555516, + "num_tokens": 177838032.0, + "rewards/accuracies": 0.69296875, + "rewards/chosen": -0.19367313385009766, + "rewards/margins": 1.2017547607421875, + "rewards/rejected": -1.39564208984375, + "step": 1060 + }, + { + "entropy": 1.1814453125, + "epoch": 0.5018614604403013, + "grad_norm": 43.66828536987305, + "learning_rate": 2.7722772277227726e-07, + "logits/chosen": -0.35000077578470384, + "logits/rejected": 0.07725395072725653, + "logps/chosen": -449.09375, + "logps/rejected": -622.125, + "loss": 0.4893054962158203, + "mean_token_accuracy": 0.7206394921988248, + "num_tokens": 179504581.0, + "rewards/accuracies": 0.72421875, + "rewards/chosen": -0.19243621826171875, + "rewards/margins": 1.165966796875, + "rewards/rejected": -1.3583648681640625, + "step": 1070 + }, + { + "entropy": 1.25322265625, + "epoch": 0.5065517544631079, + "grad_norm": 40.902252197265625, + "learning_rate": 2.7462219906201143e-07, + "logits/chosen": -0.33450535889940103, + "logits/rejected": 0.1168974359193452, + "logps/chosen": -470.7125, + "logps/rejected": -658.0875, + "loss": 0.5172897338867187, + "mean_token_accuracy": 0.7043697636574506, + "num_tokens": 181183445.0, + "rewards/accuracies": 0.7140625, + "rewards/chosen": -0.2109607696533203, + "rewards/margins": 1.235137939453125, + "rewards/rejected": -1.4460359573364259, + "step": 1080 + }, + { + "entropy": 1.3098876953125, + "epoch": 0.5112420484859145, + "grad_norm": 47.25940704345703, + "learning_rate": 2.7201667535174566e-07, + "logits/chosen": -0.29794072833065727, + "logits/rejected": 0.10716472846404641, + "logps/chosen": -498.059375, + "logps/rejected": -685.2625, + "loss": 0.526763916015625, + "mean_token_accuracy": 0.6980478201061487, + "num_tokens": 182829730.0, + "rewards/accuracies": 0.71015625, + "rewards/chosen": -0.2504600524902344, + "rewards/margins": 1.2077743530273437, + "rewards/rejected": -1.457940673828125, + "step": 1090 + }, + { + "entropy": 1.32303466796875, + "epoch": 0.515932342508721, + "grad_norm": 39.55305862426758, + "learning_rate": 2.6941115164147994e-07, + "logits/chosen": -0.36416835318792495, + "logits/rejected": 0.10446686581728493, + "logps/chosen": -481.640625, + "logps/rejected": -717.20625, + "loss": 0.5190200805664062, + "mean_token_accuracy": 0.7050384001806378, + "num_tokens": 184507519.0, + "rewards/accuracies": 0.728125, + "rewards/chosen": -0.24545488357543946, + "rewards/margins": 1.1905990600585938, + "rewards/rejected": -1.4363555908203125, + "step": 1100 + }, + { + "entropy": 1.25341796875, + "epoch": 0.5206226365315275, + "grad_norm": 39.871578216552734, + "learning_rate": 2.6680562793121416e-07, + "logits/chosen": -0.3013125689062654, + "logits/rejected": 0.10314716436853602, + "logps/chosen": -473.5625, + "logps/rejected": -651.525, + "loss": 0.5050193786621093, + "mean_token_accuracy": 0.7060678452253342, + "num_tokens": 186153120.0, + "rewards/accuracies": 0.703125, + "rewards/chosen": -0.1920490264892578, + "rewards/margins": 1.227564239501953, + "rewards/rejected": -1.4199142456054688, + "step": 1110 + }, + { + "entropy": 1.176513671875, + "epoch": 0.5253129305543341, + "grad_norm": 52.36431121826172, + "learning_rate": 2.642001042209484e-07, + "logits/chosen": -0.26450315077369746, + "logits/rejected": 0.15473802955973542, + "logps/chosen": -464.03125, + "logps/rejected": -627.96875, + "loss": 0.49540557861328127, + "mean_token_accuracy": 0.7205847401171923, + "num_tokens": 187806398.0, + "rewards/accuracies": 0.71796875, + "rewards/chosen": -0.2411958694458008, + "rewards/margins": 1.1548843383789062, + "rewards/rejected": -1.395941162109375, + "step": 1120 + }, + { + "entropy": 1.31982421875, + "epoch": 0.5300032245771407, + "grad_norm": 36.76437759399414, + "learning_rate": 2.615945805106826e-07, + "logits/chosen": -0.3149445633982369, + "logits/rejected": 0.16251844820700828, + "logps/chosen": -476.3375, + "logps/rejected": -757.70625, + "loss": 0.5259773254394531, + "mean_token_accuracy": 0.7150143172591925, + "num_tokens": 189521715.0, + "rewards/accuracies": 0.71953125, + "rewards/chosen": -0.2303621292114258, + "rewards/margins": 1.315435791015625, + "rewards/rejected": -1.5461624145507813, + "step": 1130 + }, + { + "entropy": 1.2891357421875, + "epoch": 0.5346935185999472, + "grad_norm": 70.96980285644531, + "learning_rate": 2.589890568004169e-07, + "logits/chosen": -0.3271291111287221, + "logits/rejected": 0.16675223605322737, + "logps/chosen": -500.31875, + "logps/rejected": -694.71875, + "loss": 0.5380020141601562, + "mean_token_accuracy": 0.705069973319769, + "num_tokens": 191236695.0, + "rewards/accuracies": 0.7203125, + "rewards/chosen": -0.3011770248413086, + "rewards/margins": 1.1962921142578125, + "rewards/rejected": -1.4982635498046875, + "step": 1140 + }, + { + "entropy": 1.2275634765625, + "epoch": 0.5393838126227538, + "grad_norm": 48.03555679321289, + "learning_rate": 2.5638353309015107e-07, + "logits/chosen": -0.30602316091402987, + "logits/rejected": 0.08515583556015452, + "logps/chosen": -465.7125, + "logps/rejected": -623.03125, + "loss": 0.4871429443359375, + "mean_token_accuracy": 0.7122127812355756, + "num_tokens": 192930148.0, + "rewards/accuracies": 0.71875, + "rewards/chosen": -0.2552909851074219, + "rewards/margins": 1.3664947509765626, + "rewards/rejected": -1.6219619750976562, + "step": 1150 + }, + { + "entropy": 1.312890625, + "epoch": 0.5440741066455603, + "grad_norm": 39.23597717285156, + "learning_rate": 2.5377800937988535e-07, + "logits/chosen": -0.34489304897849415, + "logits/rejected": 0.1796444057444734, + "logps/chosen": -510.19375, + "logps/rejected": -763.225, + "loss": 0.5116714477539063, + "mean_token_accuracy": 0.7016687668859959, + "num_tokens": 194613883.0, + "rewards/accuracies": 0.709375, + "rewards/chosen": -0.2514598846435547, + "rewards/margins": 1.2324905395507812, + "rewards/rejected": -1.4837749481201172, + "step": 1160 + }, + { + "entropy": 1.28082275390625, + "epoch": 0.5487644006683668, + "grad_norm": 69.99358367919922, + "learning_rate": 2.511724856696196e-07, + "logits/chosen": -0.27411585619664347, + "logits/rejected": 0.28711497107316053, + "logps/chosen": -474.628125, + "logps/rejected": -750.44375, + "loss": 0.5183349609375, + "mean_token_accuracy": 0.7180445723235607, + "num_tokens": 196335064.0, + "rewards/accuracies": 0.71953125, + "rewards/chosen": -0.289007568359375, + "rewards/margins": 1.1853988647460938, + "rewards/rejected": -1.474517822265625, + "step": 1170 + }, + { + "entropy": 1.265185546875, + "epoch": 0.5534546946911735, + "grad_norm": 36.792850494384766, + "learning_rate": 2.485669619593538e-07, + "logits/chosen": -0.3359410788666688, + "logits/rejected": 0.05803117348722396, + "logps/chosen": -470.0328125, + "logps/rejected": -689.440625, + "loss": 0.5088897705078125, + "mean_token_accuracy": 0.7110123753547668, + "num_tokens": 198019334.0, + "rewards/accuracies": 0.70234375, + "rewards/chosen": -0.2291189193725586, + "rewards/margins": 1.2735336303710938, + "rewards/rejected": -1.5025585174560547, + "step": 1180 + }, + { + "entropy": 1.111865234375, + "epoch": 0.55814498871398, + "grad_norm": 39.022945404052734, + "learning_rate": 2.45961438249088e-07, + "logits/chosen": -0.3723478462502887, + "logits/rejected": 0.07380545194565284, + "logps/chosen": -442.759375, + "logps/rejected": -580.7875, + "loss": 0.4936191558837891, + "mean_token_accuracy": 0.7239787954837084, + "num_tokens": 199692142.0, + "rewards/accuracies": 0.7171875, + "rewards/chosen": -0.2774223327636719, + "rewards/margins": 1.1883934020996094, + "rewards/rejected": -1.4662437438964844, + "step": 1190 + }, + { + "entropy": 1.2387451171875, + "epoch": 0.5628352827367865, + "grad_norm": 48.250648498535156, + "learning_rate": 2.433559145388223e-07, + "logits/chosen": -0.37183548958390833, + "logits/rejected": 0.058249644272995095, + "logps/chosen": -463.95, + "logps/rejected": -668.440625, + "loss": 0.5038101196289062, + "mean_token_accuracy": 0.7077918142080307, + "num_tokens": 201359937.0, + "rewards/accuracies": 0.71328125, + "rewards/chosen": -0.25881156921386717, + "rewards/margins": 1.1958583831787108, + "rewards/rejected": -1.4544029235839844, + "step": 1200 + }, + { + "entropy": 1.2365234375, + "epoch": 0.5675255767595931, + "grad_norm": 57.87482833862305, + "learning_rate": 2.4075039082855653e-07, + "logits/chosen": -0.3253422542766312, + "logits/rejected": 0.11797782865500077, + "logps/chosen": -433.44375, + "logps/rejected": -660.53125, + "loss": 0.5135528564453125, + "mean_token_accuracy": 0.7127903729677201, + "num_tokens": 203050869.0, + "rewards/accuracies": 0.69140625, + "rewards/chosen": -0.21519794464111328, + "rewards/margins": 1.4135284423828125, + "rewards/rejected": -1.630401611328125, + "step": 1210 + }, + { + "entropy": 1.23800048828125, + "epoch": 0.5722158707823997, + "grad_norm": 43.60979080200195, + "learning_rate": 2.3814486711829076e-07, + "logits/chosen": -0.4246529642592443, + "logits/rejected": 0.08977297602859086, + "logps/chosen": -451.81875, + "logps/rejected": -692.68125, + "loss": 0.4846954345703125, + "mean_token_accuracy": 0.7116952534765005, + "num_tokens": 204729703.0, + "rewards/accuracies": 0.72578125, + "rewards/chosen": -0.3041818618774414, + "rewards/margins": 1.2798492431640625, + "rewards/rejected": -1.5844802856445312, + "step": 1220 + }, + { + "entropy": 1.32520751953125, + "epoch": 0.5769061648052062, + "grad_norm": 52.88212585449219, + "learning_rate": 2.35539343408025e-07, + "logits/chosen": -0.4144738956031821, + "logits/rejected": 0.1442082849155881, + "logps/chosen": -495.73125, + "logps/rejected": -764.740625, + "loss": 0.505401611328125, + "mean_token_accuracy": 0.707651786506176, + "num_tokens": 206446007.0, + "rewards/accuracies": 0.7171875, + "rewards/chosen": -0.37430610656738283, + "rewards/margins": 1.3635223388671875, + "rewards/rejected": -1.738238525390625, + "step": 1230 + }, + { + "entropy": 1.250390625, + "epoch": 0.5815964588280128, + "grad_norm": 47.2264404296875, + "learning_rate": 2.3293381969775924e-07, + "logits/chosen": -0.34146382107676954, + "logits/rejected": 0.16612224711888873, + "logps/chosen": -488.14375, + "logps/rejected": -695.575, + "loss": 0.510650634765625, + "mean_token_accuracy": 0.7113540126010776, + "num_tokens": 208151679.0, + "rewards/accuracies": 0.71171875, + "rewards/chosen": -0.2607236862182617, + "rewards/margins": 1.2523696899414063, + "rewards/rejected": -1.5134803771972656, + "step": 1240 + }, + { + "entropy": 1.2435302734375, + "epoch": 0.5862867528508193, + "grad_norm": 43.1168327331543, + "learning_rate": 2.303282959874935e-07, + "logits/chosen": -0.28546225309033735, + "logits/rejected": 0.2084978743788679, + "logps/chosen": -489.084375, + "logps/rejected": -698.959375, + "loss": 0.4846649169921875, + "mean_token_accuracy": 0.7133546834811568, + "num_tokens": 209904565.0, + "rewards/accuracies": 0.7296875, + "rewards/chosen": -0.23052234649658204, + "rewards/margins": 1.3557510375976562, + "rewards/rejected": -1.586376953125, + "step": 1250 + }, + { + "entropy": 1.304833984375, + "epoch": 0.5909770468736258, + "grad_norm": 46.38766098022461, + "learning_rate": 2.2772277227722772e-07, + "logits/chosen": -0.3320207102770688, + "logits/rejected": 0.15268373542131383, + "logps/chosen": -497.2, + "logps/rejected": -721.1125, + "loss": 0.4992431640625, + "mean_token_accuracy": 0.7041572637856006, + "num_tokens": 211634743.0, + "rewards/accuracies": 0.721875, + "rewards/chosen": -0.23194313049316406, + "rewards/margins": 1.245294189453125, + "rewards/rejected": -1.477325439453125, + "step": 1260 + }, + { + "entropy": 1.3220458984375, + "epoch": 0.5956673408964325, + "grad_norm": 49.88144302368164, + "learning_rate": 2.2511724856696197e-07, + "logits/chosen": -0.36437243560616833, + "logits/rejected": 0.3099788081189955, + "logps/chosen": -487.084375, + "logps/rejected": -765.0875, + "loss": 0.5212089538574218, + "mean_token_accuracy": 0.6997947275638581, + "num_tokens": 213353842.0, + "rewards/accuracies": 0.7046875, + "rewards/chosen": -0.24524993896484376, + "rewards/margins": 1.2819038391113282, + "rewards/rejected": -1.5273468017578125, + "step": 1270 + }, + { + "entropy": 1.2190673828125, + "epoch": 0.600357634919239, + "grad_norm": 58.774261474609375, + "learning_rate": 2.225117248566962e-07, + "logits/chosen": -0.2944029906126194, + "logits/rejected": 0.12548050740456665, + "logps/chosen": -478.509375, + "logps/rejected": -663.2625, + "loss": 0.49728050231933596, + "mean_token_accuracy": 0.7116608984768391, + "num_tokens": 215023462.0, + "rewards/accuracies": 0.69609375, + "rewards/chosen": -0.23260726928710937, + "rewards/margins": 1.20821533203125, + "rewards/rejected": -1.4408126831054688, + "step": 1280 + }, + { + "entropy": 1.312060546875, + "epoch": 0.6050479289420455, + "grad_norm": 50.618873596191406, + "learning_rate": 2.1990620114643042e-07, + "logits/chosen": -0.4199054390668423, + "logits/rejected": 0.18468027054402908, + "logps/chosen": -486.25, + "logps/rejected": -709.20625, + "loss": 0.5122344970703125, + "mean_token_accuracy": 0.6997991988435388, + "num_tokens": 216700271.0, + "rewards/accuracies": 0.6953125, + "rewards/chosen": -0.21283988952636718, + "rewards/margins": 1.145751953125, + "rewards/rejected": -1.3585968017578125, + "step": 1290 + }, + { + "entropy": 1.25576171875, + "epoch": 0.6097382229648521, + "grad_norm": 52.30256652832031, + "learning_rate": 2.1730067743616465e-07, + "logits/chosen": -0.33569222926382486, + "logits/rejected": 0.1868876063062059, + "logps/chosen": -454.11875, + "logps/rejected": -698.68125, + "loss": 0.48102340698242185, + "mean_token_accuracy": 0.7133908785879612, + "num_tokens": 218349883.0, + "rewards/accuracies": 0.71640625, + "rewards/chosen": -0.17691356539726258, + "rewards/margins": 1.347509765625, + "rewards/rejected": -1.5247505187988282, + "step": 1300 + }, + { + "entropy": 1.25462646484375, + "epoch": 0.6144285169876587, + "grad_norm": 64.008544921875, + "learning_rate": 2.1469515372589887e-07, + "logits/chosen": -0.38799636342554056, + "logits/rejected": 0.05200148625905195, + "logps/chosen": -492.275, + "logps/rejected": -644.978125, + "loss": 0.5071174621582031, + "mean_token_accuracy": 0.7041159149259328, + "num_tokens": 220019139.0, + "rewards/accuracies": 0.69921875, + "rewards/chosen": -0.21364364624023438, + "rewards/margins": 1.1556907653808595, + "rewards/rejected": -1.3692604064941407, + "step": 1310 + }, + { + "entropy": 1.2711181640625, + "epoch": 0.6191188110104652, + "grad_norm": 48.48934555053711, + "learning_rate": 2.1208963001563313e-07, + "logits/chosen": -0.3561808660204896, + "logits/rejected": 0.13831664785273107, + "logps/chosen": -500.71875, + "logps/rejected": -694.2375, + "loss": 0.5029388427734375, + "mean_token_accuracy": 0.7105904638767242, + "num_tokens": 221722944.0, + "rewards/accuracies": 0.69375, + "rewards/chosen": -0.23125057220458983, + "rewards/margins": 1.2435516357421874, + "rewards/rejected": -1.4749588012695312, + "step": 1320 + }, + { + "entropy": 1.19822998046875, + "epoch": 0.6238091050332718, + "grad_norm": 42.350685119628906, + "learning_rate": 2.0948410630536735e-07, + "logits/chosen": -0.37555533940911084, + "logits/rejected": -0.011265530235249926, + "logps/chosen": -457.528125, + "logps/rejected": -591.6, + "loss": 0.5330169677734375, + "mean_token_accuracy": 0.7136769216507673, + "num_tokens": 223404308.0, + "rewards/accuracies": 0.6921875, + "rewards/chosen": -0.3046894073486328, + "rewards/margins": 1.0406204223632813, + "rewards/rejected": -1.3457130432128905, + "step": 1330 + }, + { + "entropy": 1.22252197265625, + "epoch": 0.6284993990560783, + "grad_norm": 45.17627716064453, + "learning_rate": 2.068785825951016e-07, + "logits/chosen": -0.28848142084641326, + "logits/rejected": 0.12333503271507404, + "logps/chosen": -498.05, + "logps/rejected": -652.503125, + "loss": 0.4917594909667969, + "mean_token_accuracy": 0.7163054622709751, + "num_tokens": 225127215.0, + "rewards/accuracies": 0.7203125, + "rewards/chosen": -0.24520225524902345, + "rewards/margins": 1.2569358825683594, + "rewards/rejected": -1.5024093627929687, + "step": 1340 + }, + { + "entropy": 1.3197998046875, + "epoch": 0.6331896930788848, + "grad_norm": 78.06428527832031, + "learning_rate": 2.0427305888483583e-07, + "logits/chosen": -0.363285197439468, + "logits/rejected": 0.12140866192187372, + "logps/chosen": -479.86875, + "logps/rejected": -726.678125, + "loss": 0.5196853637695312, + "mean_token_accuracy": 0.6993409872055054, + "num_tokens": 226798538.0, + "rewards/accuracies": 0.71171875, + "rewards/chosen": -0.2906646728515625, + "rewards/margins": 1.2201629638671876, + "rewards/rejected": -1.51029052734375, + "step": 1350 + }, + { + "entropy": 1.263818359375, + "epoch": 0.6378799871016915, + "grad_norm": 45.774662017822266, + "learning_rate": 2.0166753517457009e-07, + "logits/chosen": -0.3178116434676858, + "logits/rejected": 0.13615178149001414, + "logps/chosen": -483.6, + "logps/rejected": -678.51875, + "loss": 0.5075515747070313, + "mean_token_accuracy": 0.7105112101882696, + "num_tokens": 228525211.0, + "rewards/accuracies": 0.71953125, + "rewards/chosen": -0.22887954711914063, + "rewards/margins": 1.2723709106445313, + "rewards/rejected": -1.5004180908203124, + "step": 1360 + }, + { + "entropy": 1.28734130859375, + "epoch": 0.642570281124498, + "grad_norm": 45.82160568237305, + "learning_rate": 1.990620114643043e-07, + "logits/chosen": -0.2266978024313957, + "logits/rejected": 0.3092070634916513, + "logps/chosen": -473.2625, + "logps/rejected": -772.3, + "loss": 0.49246978759765625, + "mean_token_accuracy": 0.7207242472097277, + "num_tokens": 230277952.0, + "rewards/accuracies": 0.72734375, + "rewards/chosen": -0.18862266540527345, + "rewards/margins": 1.2539337158203125, + "rewards/rejected": -1.4429153442382812, + "step": 1370 + }, + { + "entropy": 1.2372802734375, + "epoch": 0.6472605751473045, + "grad_norm": 53.567955017089844, + "learning_rate": 1.9645648775403857e-07, + "logits/chosen": -0.3122498323546022, + "logits/rejected": -0.029102683684336005, + "logps/chosen": -456.4125, + "logps/rejected": -621.05625, + "loss": 0.5261543273925782, + "mean_token_accuracy": 0.7144070915877819, + "num_tokens": 231910230.0, + "rewards/accuracies": 0.70234375, + "rewards/chosen": -0.16479625701904296, + "rewards/margins": 1.0987884521484375, + "rewards/rejected": -1.263460922241211, + "step": 1380 + }, + { + "entropy": 1.25101318359375, + "epoch": 0.6519508691701111, + "grad_norm": 54.65653610229492, + "learning_rate": 1.938509640437728e-07, + "logits/chosen": -0.2595864928279747, + "logits/rejected": 0.17200187501163847, + "logps/chosen": -457.83125, + "logps/rejected": -694.93125, + "loss": 0.5121467590332032, + "mean_token_accuracy": 0.7173424143344164, + "num_tokens": 233587330.0, + "rewards/accuracies": 0.7234375, + "rewards/chosen": -0.223736572265625, + "rewards/margins": 1.3258369445800782, + "rewards/rejected": -1.5491180419921875, + "step": 1390 + }, + { + "entropy": 1.2558837890625, + "epoch": 0.6566411631929177, + "grad_norm": 50.95151138305664, + "learning_rate": 1.9124544033350704e-07, + "logits/chosen": -0.2936878858134837, + "logits/rejected": 0.12141680482417419, + "logps/chosen": -477.040625, + "logps/rejected": -639.909375, + "loss": 0.5095062255859375, + "mean_token_accuracy": 0.708953882008791, + "num_tokens": 235270410.0, + "rewards/accuracies": 0.71015625, + "rewards/chosen": -0.20118942260742187, + "rewards/margins": 1.2124801635742188, + "rewards/rejected": -1.4136474609375, + "step": 1400 + }, + { + "entropy": 1.3095947265625, + "epoch": 0.6613314572157242, + "grad_norm": 40.7271728515625, + "learning_rate": 1.8863991662324127e-07, + "logits/chosen": -0.34175182540570026, + "logits/rejected": 0.13167951761138544, + "logps/chosen": -459.971875, + "logps/rejected": -683.640625, + "loss": 0.5016754150390625, + "mean_token_accuracy": 0.7081727286800742, + "num_tokens": 236914943.0, + "rewards/accuracies": 0.71640625, + "rewards/chosen": -0.19021434783935548, + "rewards/margins": 1.188072967529297, + "rewards/rejected": -1.378137969970703, + "step": 1410 + }, + { + "entropy": 1.2114013671875, + "epoch": 0.6660217512385308, + "grad_norm": 39.70977783203125, + "learning_rate": 1.8603439291297552e-07, + "logits/chosen": -0.24609394416242006, + "logits/rejected": 0.1799542514096824, + "logps/chosen": -494.9625, + "logps/rejected": -643.74375, + "loss": 0.5288177490234375, + "mean_token_accuracy": 0.7189813105389475, + "num_tokens": 238709572.0, + "rewards/accuracies": 0.70625, + "rewards/chosen": -0.2049093246459961, + "rewards/margins": 1.2056396484375, + "rewards/rejected": -1.4105159759521484, + "step": 1420 + }, + { + "entropy": 1.21826171875, + "epoch": 0.6707120452613373, + "grad_norm": 66.35521697998047, + "learning_rate": 1.8342886920270972e-07, + "logits/chosen": -0.32402960394939234, + "logits/rejected": 0.06577563256399063, + "logps/chosen": -470.8625, + "logps/rejected": -613.003125, + "loss": 0.5125747680664062, + "mean_token_accuracy": 0.7137641754001379, + "num_tokens": 240329577.0, + "rewards/accuracies": 0.721875, + "rewards/chosen": -0.28786163330078124, + "rewards/margins": 1.20198974609375, + "rewards/rejected": -1.490332794189453, + "step": 1430 + }, + { + "entropy": 1.2763671875, + "epoch": 0.6754023392841438, + "grad_norm": 59.820579528808594, + "learning_rate": 1.8082334549244395e-07, + "logits/chosen": -0.2621199295266179, + "logits/rejected": 0.21548721230861387, + "logps/chosen": -488.2375, + "logps/rejected": -711.73125, + "loss": 0.5426666259765625, + "mean_token_accuracy": 0.7102572401985526, + "num_tokens": 242037983.0, + "rewards/accuracies": 0.6796875, + "rewards/chosen": -0.2579639434814453, + "rewards/margins": 1.1012718200683593, + "rewards/rejected": -1.3592147827148438, + "step": 1440 + }, + { + "entropy": 1.2553955078125, + "epoch": 0.6800926333069505, + "grad_norm": 39.72897720336914, + "learning_rate": 1.782178217821782e-07, + "logits/chosen": -0.3045310204445476, + "logits/rejected": 0.11346873189208409, + "logps/chosen": -502.8375, + "logps/rejected": -687.9125, + "loss": 0.5121414184570312, + "mean_token_accuracy": 0.7075636066496372, + "num_tokens": 243740239.0, + "rewards/accuracies": 0.7234375, + "rewards/chosen": -0.26166362762451173, + "rewards/margins": 1.3748176574707032, + "rewards/rejected": -1.6358539581298828, + "step": 1450 + }, + { + "entropy": 1.2446044921875, + "epoch": 0.684782927329757, + "grad_norm": 42.146484375, + "learning_rate": 1.7561229807191243e-07, + "logits/chosen": -0.3356412574221423, + "logits/rejected": 0.10238671794143098, + "logps/chosen": -451.0875, + "logps/rejected": -697.11875, + "loss": 0.5116485595703125, + "mean_token_accuracy": 0.7145647119730711, + "num_tokens": 245400678.0, + "rewards/accuracies": 0.72109375, + "rewards/chosen": -0.2481781005859375, + "rewards/margins": 1.1638092041015624, + "rewards/rejected": -1.4118682861328125, + "step": 1460 + }, + { + "entropy": 1.185888671875, + "epoch": 0.6894732213525635, + "grad_norm": 47.214019775390625, + "learning_rate": 1.7300677436164668e-07, + "logits/chosen": -0.39723945328086835, + "logits/rejected": -0.04166929725172916, + "logps/chosen": -467.621875, + "logps/rejected": -568.09375, + "loss": 0.4959739685058594, + "mean_token_accuracy": 0.7094458827748895, + "num_tokens": 247054384.0, + "rewards/accuracies": 0.703125, + "rewards/chosen": -0.20417518615722657, + "rewards/margins": 1.1270927429199218, + "rewards/rejected": -1.3313568115234375, + "step": 1470 + }, + { + "entropy": 1.2665283203125, + "epoch": 0.6941635153753701, + "grad_norm": 53.28609848022461, + "learning_rate": 1.704012506513809e-07, + "logits/chosen": -0.3926752625519089, + "logits/rejected": 0.12300171587715411, + "logps/chosen": -490.3625, + "logps/rejected": -751.45, + "loss": 0.5226806640625, + "mean_token_accuracy": 0.7115387700498104, + "num_tokens": 248780570.0, + "rewards/accuracies": 0.709375, + "rewards/chosen": -0.31193199157714846, + "rewards/margins": 1.3558914184570312, + "rewards/rejected": -1.6674148559570312, + "step": 1480 + }, + { + "entropy": 1.3110107421875, + "epoch": 0.6988538093981767, + "grad_norm": 44.8239860534668, + "learning_rate": 1.6779572694111516e-07, + "logits/chosen": -0.4324692173902081, + "logits/rejected": 0.07500056479665548, + "logps/chosen": -438.93125, + "logps/rejected": -697.46875, + "loss": 0.4912841796875, + "mean_token_accuracy": 0.7042546521872282, + "num_tokens": 250392447.0, + "rewards/accuracies": 0.73125, + "rewards/chosen": -0.21766271591186523, + "rewards/margins": 1.302496337890625, + "rewards/rejected": -1.5202301025390625, + "step": 1490 + }, + { + "entropy": 1.2731201171875, + "epoch": 0.7035441034209832, + "grad_norm": 59.22135543823242, + "learning_rate": 1.651902032308494e-07, + "logits/chosen": -0.3529709127225384, + "logits/rejected": 0.1258731407236429, + "logps/chosen": -452.975, + "logps/rejected": -722.65625, + "loss": 0.49285736083984377, + "mean_token_accuracy": 0.7159934408962727, + "num_tokens": 252045693.0, + "rewards/accuracies": 0.69765625, + "rewards/chosen": -0.19012088775634767, + "rewards/margins": 1.233636474609375, + "rewards/rejected": -1.4240127563476563, + "step": 1500 + }, + { + "entropy": 1.21953125, + "epoch": 0.7082343974437898, + "grad_norm": 67.9064712524414, + "learning_rate": 1.6258467952058364e-07, + "logits/chosen": -0.355636732334131, + "logits/rejected": 0.06523703115240106, + "logps/chosen": -484.0, + "logps/rejected": -613.578125, + "loss": 0.5271133422851563, + "mean_token_accuracy": 0.7069945391267538, + "num_tokens": 253694458.0, + "rewards/accuracies": 0.70078125, + "rewards/chosen": -0.21510238647460939, + "rewards/margins": 1.0788116455078125, + "rewards/rejected": -1.2938308715820312, + "step": 1510 + }, + { + "entropy": 1.2014404296875, + "epoch": 0.7129246914665963, + "grad_norm": 40.9355583190918, + "learning_rate": 1.5997915581031787e-07, + "logits/chosen": -0.40950678159563625, + "logits/rejected": 0.01910210709722237, + "logps/chosen": -454.46875, + "logps/rejected": -634.725, + "loss": 0.5015480041503906, + "mean_token_accuracy": 0.7128090810030698, + "num_tokens": 255410705.0, + "rewards/accuracies": 0.7109375, + "rewards/chosen": -0.24631481170654296, + "rewards/margins": 1.258221435546875, + "rewards/rejected": -1.5049667358398438, + "step": 1520 + }, + { + "entropy": 1.241748046875, + "epoch": 0.7176149854894028, + "grad_norm": 43.45293426513672, + "learning_rate": 1.5737363210005212e-07, + "logits/chosen": -0.3803298971013801, + "logits/rejected": -0.022503258323848245, + "logps/chosen": -469.95625, + "logps/rejected": -660.88125, + "loss": 0.5067474365234375, + "mean_token_accuracy": 0.7098428962752223, + "num_tokens": 257032747.0, + "rewards/accuracies": 0.7046875, + "rewards/chosen": -0.20492134094238282, + "rewards/margins": 1.2111480712890625, + "rewards/rejected": -1.4155357360839844, + "step": 1530 + }, + { + "entropy": 1.2524169921875, + "epoch": 0.7223052795122095, + "grad_norm": 46.686073303222656, + "learning_rate": 1.5476810838978635e-07, + "logits/chosen": -0.409221722079539, + "logits/rejected": 0.037211792533006396, + "logps/chosen": -465.66875, + "logps/rejected": -655.08515625, + "loss": 0.49529037475585935, + "mean_token_accuracy": 0.7079074520617723, + "num_tokens": 258717701.0, + "rewards/accuracies": 0.73125, + "rewards/chosen": -0.18844895362854003, + "rewards/margins": 1.2114707946777343, + "rewards/rejected": -1.4001953125, + "step": 1540 + }, + { + "entropy": 1.20712890625, + "epoch": 0.726995573535016, + "grad_norm": 41.296077728271484, + "learning_rate": 1.521625846795206e-07, + "logits/chosen": -0.31212731107450276, + "logits/rejected": 0.11191561789900543, + "logps/chosen": -480.353125, + "logps/rejected": -633.00625, + "loss": 0.503009033203125, + "mean_token_accuracy": 0.717727231234312, + "num_tokens": 260394661.0, + "rewards/accuracies": 0.7265625, + "rewards/chosen": -0.2229745864868164, + "rewards/margins": 1.2388389587402344, + "rewards/rejected": -1.461846160888672, + "step": 1550 + }, + { + "entropy": 1.279541015625, + "epoch": 0.7316858675578225, + "grad_norm": 40.81024932861328, + "learning_rate": 1.495570609692548e-07, + "logits/chosen": -0.36629913463706937, + "logits/rejected": 0.12231380401378948, + "logps/chosen": -463.809375, + "logps/rejected": -651.1375, + "loss": 0.5072137832641601, + "mean_token_accuracy": 0.70716253798455, + "num_tokens": 262044098.0, + "rewards/accuracies": 0.71484375, + "rewards/chosen": -0.16593360900878906, + "rewards/margins": 1.2185470581054687, + "rewards/rejected": -1.384647750854492, + "step": 1560 + }, + { + "entropy": 1.2444580078125, + "epoch": 0.7363761615806291, + "grad_norm": 47.42740249633789, + "learning_rate": 1.4695153725898905e-07, + "logits/chosen": -0.33029111726362215, + "logits/rejected": 0.16555770502337477, + "logps/chosen": -491.6, + "logps/rejected": -660.21875, + "loss": 0.526171875, + "mean_token_accuracy": 0.7089153520762921, + "num_tokens": 263781303.0, + "rewards/accuracies": 0.7046875, + "rewards/chosen": -0.20786285400390625, + "rewards/margins": 1.1246536254882813, + "rewards/rejected": -1.33277587890625, + "step": 1570 + }, + { + "entropy": 1.2452880859375, + "epoch": 0.7410664556034356, + "grad_norm": 36.83119201660156, + "learning_rate": 1.4434601354872328e-07, + "logits/chosen": -0.35121305578205475, + "logits/rejected": 0.11945853415718628, + "logps/chosen": -461.51875, + "logps/rejected": -655.359375, + "loss": 0.482196044921875, + "mean_token_accuracy": 0.7106880776584148, + "num_tokens": 265433617.0, + "rewards/accuracies": 0.72265625, + "rewards/chosen": -0.16603565216064453, + "rewards/margins": 1.304248046875, + "rewards/rejected": -1.470250701904297, + "step": 1580 + }, + { + "entropy": 1.2256103515625, + "epoch": 0.7457567496262422, + "grad_norm": 39.94232177734375, + "learning_rate": 1.417404898384575e-07, + "logits/chosen": -0.3938522607414349, + "logits/rejected": 0.11833697097822693, + "logps/chosen": -456.45625, + "logps/rejected": -635.05, + "loss": 0.5009937286376953, + "mean_token_accuracy": 0.7109608668833971, + "num_tokens": 267110073.0, + "rewards/accuracies": 0.7171875, + "rewards/chosen": -0.17589912414550782, + "rewards/margins": 1.1410980224609375, + "rewards/rejected": -1.3171512603759765, + "step": 1590 + }, + { + "entropy": 1.2601318359375, + "epoch": 0.7504470436490488, + "grad_norm": 43.43891525268555, + "learning_rate": 1.3913496612819176e-07, + "logits/chosen": -0.31867154075075216, + "logits/rejected": 0.07912379150382028, + "logps/chosen": -460.015625, + "logps/rejected": -635.6125, + "loss": 0.5057296752929688, + "mean_token_accuracy": 0.7127839136868716, + "num_tokens": 268708942.0, + "rewards/accuracies": 0.72109375, + "rewards/chosen": -0.19757213592529296, + "rewards/margins": 1.215704345703125, + "rewards/rejected": -1.413494873046875, + "step": 1600 + }, + { + "entropy": 1.2163818359375, + "epoch": 0.7551373376718553, + "grad_norm": 44.50661087036133, + "learning_rate": 1.3652944241792598e-07, + "logits/chosen": -0.414186569736156, + "logits/rejected": -0.032987588264573646, + "logps/chosen": -501.94375, + "logps/rejected": -625.9125, + "loss": 0.5188003540039062, + "mean_token_accuracy": 0.7055005624890327, + "num_tokens": 270406707.0, + "rewards/accuracies": 0.7, + "rewards/chosen": -0.21747322082519532, + "rewards/margins": 1.1645050048828125, + "rewards/rejected": -1.3811737060546876, + "step": 1610 + }, + { + "entropy": 1.2386474609375, + "epoch": 0.7598276316946618, + "grad_norm": 40.76424026489258, + "learning_rate": 1.3392391870766024e-07, + "logits/chosen": -0.25317166552302806, + "logits/rejected": 0.25054843549323225, + "logps/chosen": -448.25625, + "logps/rejected": -678.025, + "loss": 0.498260498046875, + "mean_token_accuracy": 0.7153599586337804, + "num_tokens": 272057284.0, + "rewards/accuracies": 0.71328125, + "rewards/chosen": -0.15067138671875, + "rewards/margins": 1.1843612670898438, + "rewards/rejected": -1.3350822448730468, + "step": 1620 + }, + { + "entropy": 1.3083251953125, + "epoch": 0.7645179257174685, + "grad_norm": 53.002323150634766, + "learning_rate": 1.3131839499739446e-07, + "logits/chosen": -0.3471587077953908, + "logits/rejected": 0.19919105227987713, + "logps/chosen": -494.4125, + "logps/rejected": -728.68125, + "loss": 0.4993560791015625, + "mean_token_accuracy": 0.7017248619347811, + "num_tokens": 273777667.0, + "rewards/accuracies": 0.7359375, + "rewards/chosen": -0.22601699829101562, + "rewards/margins": 1.2609451293945313, + "rewards/rejected": -1.4867431640625, + "step": 1630 + }, + { + "entropy": 1.24683837890625, + "epoch": 0.769208219740275, + "grad_norm": 58.614864349365234, + "learning_rate": 1.2871287128712872e-07, + "logits/chosen": -0.35487965619433126, + "logits/rejected": 0.08004843726996642, + "logps/chosen": -459.94375, + "logps/rejected": -656.10625, + "loss": 0.4921421051025391, + "mean_token_accuracy": 0.7122417978942395, + "num_tokens": 275509944.0, + "rewards/accuracies": 0.73125, + "rewards/chosen": -0.21575851440429689, + "rewards/margins": 1.220342254638672, + "rewards/rejected": -1.435931396484375, + "step": 1640 + }, + { + "entropy": 1.2380615234375, + "epoch": 0.7738985137630815, + "grad_norm": 46.248634338378906, + "learning_rate": 1.2610734757686294e-07, + "logits/chosen": -0.378603477523175, + "logits/rejected": 0.0143330288482811, + "logps/chosen": -473.325, + "logps/rejected": -637.475, + "loss": 0.5132255554199219, + "mean_token_accuracy": 0.7066951882094145, + "num_tokens": 277153043.0, + "rewards/accuracies": 0.721875, + "rewards/chosen": -0.2193842887878418, + "rewards/margins": 1.1281280517578125, + "rewards/rejected": -1.3469314575195312, + "step": 1650 + }, + { + "entropy": 1.2247314453125, + "epoch": 0.7785888077858881, + "grad_norm": 49.24350357055664, + "learning_rate": 1.2350182386659717e-07, + "logits/chosen": -0.41227246676183515, + "logits/rejected": 0.047900230891089454, + "logps/chosen": -448.90625, + "logps/rejected": -652.575, + "loss": 0.49638519287109373, + "mean_token_accuracy": 0.707376803830266, + "num_tokens": 278795974.0, + "rewards/accuracies": 0.71796875, + "rewards/chosen": -0.19539546966552734, + "rewards/margins": 1.2829010009765625, + "rewards/rejected": -1.47823486328125, + "step": 1660 + }, + { + "entropy": 1.1978515625, + "epoch": 0.7832791018086946, + "grad_norm": 48.88486862182617, + "learning_rate": 1.2089630015633142e-07, + "logits/chosen": -0.34792149701180813, + "logits/rejected": 0.0740143644661438, + "logps/chosen": -448.4125, + "logps/rejected": -628.1125, + "loss": 0.49319000244140626, + "mean_token_accuracy": 0.7200800001621246, + "num_tokens": 280435349.0, + "rewards/accuracies": 0.734375, + "rewards/chosen": -0.17760887145996093, + "rewards/margins": 1.242255401611328, + "rewards/rejected": -1.419854736328125, + "step": 1670 + }, + { + "entropy": 1.205029296875, + "epoch": 0.7879693958315012, + "grad_norm": 43.86332702636719, + "learning_rate": 1.1829077644606565e-07, + "logits/chosen": -0.29688106453686836, + "logits/rejected": 0.18034652960629277, + "logps/chosen": -475.58125, + "logps/rejected": -648.65, + "loss": 0.48222007751464846, + "mean_token_accuracy": 0.714630151540041, + "num_tokens": 282172052.0, + "rewards/accuracies": 0.72734375, + "rewards/chosen": -0.19398794174194336, + "rewards/margins": 1.296198272705078, + "rewards/rejected": -1.490453338623047, + "step": 1680 + }, + { + "entropy": 1.28875732421875, + "epoch": 0.7926596898543078, + "grad_norm": 44.04449462890625, + "learning_rate": 1.1568525273579989e-07, + "logits/chosen": -0.3517104921207329, + "logits/rejected": 0.10797833504461599, + "logps/chosen": -506.24375, + "logps/rejected": -716.54375, + "loss": 0.5166778564453125, + "mean_token_accuracy": 0.7019935846328735, + "num_tokens": 283802991.0, + "rewards/accuracies": 0.70703125, + "rewards/chosen": -0.24034361839294432, + "rewards/margins": 1.193011474609375, + "rewards/rejected": -1.4332107543945312, + "step": 1690 + }, + { + "entropy": 1.2384033203125, + "epoch": 0.7973499838771143, + "grad_norm": 42.062034606933594, + "learning_rate": 1.1307972902553413e-07, + "logits/chosen": -0.3546815150553647, + "logits/rejected": 0.009532807889907468, + "logps/chosen": -462.965625, + "logps/rejected": -640.890625, + "loss": 0.4919563293457031, + "mean_token_accuracy": 0.7082353591918945, + "num_tokens": 285483282.0, + "rewards/accuracies": 0.74140625, + "rewards/chosen": -0.19779624938964843, + "rewards/margins": 1.1956352233886718, + "rewards/rejected": -1.3938270568847657, + "step": 1700 + }, + { + "entropy": 1.23271484375, + "epoch": 0.8020402778999208, + "grad_norm": 44.65546798706055, + "learning_rate": 1.1047420531526837e-07, + "logits/chosen": -0.3493073648480609, + "logits/rejected": 0.07239800461008125, + "logps/chosen": -459.8, + "logps/rejected": -687.525, + "loss": 0.5149833679199218, + "mean_token_accuracy": 0.7182338822633028, + "num_tokens": 287194708.0, + "rewards/accuracies": 0.70859375, + "rewards/chosen": -0.2390056610107422, + "rewards/margins": 1.19747314453125, + "rewards/rejected": -1.4365234375, + "step": 1710 + }, + { + "entropy": 1.2440673828125, + "epoch": 0.8067305719227275, + "grad_norm": 44.430301666259766, + "learning_rate": 1.078686816050026e-07, + "logits/chosen": -0.33688168437130167, + "logits/rejected": 0.22037569268858842, + "logps/chosen": -453.78125, + "logps/rejected": -688.84375, + "loss": 0.46872520446777344, + "mean_token_accuracy": 0.7180781122297049, + "num_tokens": 288892457.0, + "rewards/accuracies": 0.7390625, + "rewards/chosen": -0.1388528823852539, + "rewards/margins": 1.2692779541015624, + "rewards/rejected": -1.4086166381835938, + "step": 1720 + }, + { + "entropy": 1.2156494140625, + "epoch": 0.811420865945534, + "grad_norm": 52.477691650390625, + "learning_rate": 1.0526315789473683e-07, + "logits/chosen": -0.34269354405153873, + "logits/rejected": 0.033022382885200416, + "logps/chosen": -451.6875, + "logps/rejected": -617.4875, + "loss": 0.4788845062255859, + "mean_token_accuracy": 0.7164073050022125, + "num_tokens": 290522032.0, + "rewards/accuracies": 0.72734375, + "rewards/chosen": -0.18807697296142578, + "rewards/margins": 1.2641963958740234, + "rewards/rejected": -1.4524368286132812, + "step": 1730 + }, + { + "entropy": 1.21414794921875, + "epoch": 0.8161111599683405, + "grad_norm": 41.159706115722656, + "learning_rate": 1.0265763418447107e-07, + "logits/chosen": -0.3360969872494596, + "logits/rejected": 0.1577603664545836, + "logps/chosen": -471.64375, + "logps/rejected": -631.603125, + "loss": 0.4908496856689453, + "mean_token_accuracy": 0.7176168402656913, + "num_tokens": 292202781.0, + "rewards/accuracies": 0.7140625, + "rewards/chosen": -0.1997955322265625, + "rewards/margins": 1.258979034423828, + "rewards/rejected": -1.4585357666015626, + "step": 1740 + }, + { + "entropy": 1.2014404296875, + "epoch": 0.8208014539911471, + "grad_norm": 44.04994583129883, + "learning_rate": 1.0005211047420531e-07, + "logits/chosen": -0.35303433846999394, + "logits/rejected": 0.0034600471369497406, + "logps/chosen": -498.23125, + "logps/rejected": -624.215625, + "loss": 0.5030181884765625, + "mean_token_accuracy": 0.7130729466676712, + "num_tokens": 293904790.0, + "rewards/accuracies": 0.73828125, + "rewards/chosen": -0.2209268569946289, + "rewards/margins": 1.241543960571289, + "rewards/rejected": -1.4627880096435546, + "step": 1750 + }, + { + "entropy": 1.2294189453125, + "epoch": 0.8254917480139536, + "grad_norm": 39.74637222290039, + "learning_rate": 9.744658676393955e-08, + "logits/chosen": -0.3580387652304702, + "logits/rejected": 0.07639393565541747, + "logps/chosen": -462.471875, + "logps/rejected": -637.53125, + "loss": 0.5074844360351562, + "mean_token_accuracy": 0.712321732006967, + "num_tokens": 295577189.0, + "rewards/accuracies": 0.7, + "rewards/chosen": -0.18819160461425782, + "rewards/margins": 1.1423080444335938, + "rewards/rejected": -1.3303741455078124, + "step": 1760 + }, + { + "entropy": 1.31600341796875, + "epoch": 0.8301820420367602, + "grad_norm": 57.74708557128906, + "learning_rate": 9.484106305367379e-08, + "logits/chosen": -0.347444101216201, + "logits/rejected": 0.1768859150063874, + "logps/chosen": -539.6375, + "logps/rejected": -739.56875, + "loss": 0.5145782470703125, + "mean_token_accuracy": 0.6992437416687608, + "num_tokens": 297311062.0, + "rewards/accuracies": 0.721875, + "rewards/chosen": -0.24378433227539062, + "rewards/margins": 1.293054962158203, + "rewards/rejected": -1.536578369140625, + "step": 1770 + }, + { + "entropy": 1.202099609375, + "epoch": 0.8348723360595668, + "grad_norm": 69.78555297851562, + "learning_rate": 9.223553934340803e-08, + "logits/chosen": -0.3747393354513525, + "logits/rejected": 0.10190953988080428, + "logps/chosen": -457.48125, + "logps/rejected": -642.8875, + "loss": 0.4973968505859375, + "mean_token_accuracy": 0.7205143462866545, + "num_tokens": 298981777.0, + "rewards/accuracies": 0.72578125, + "rewards/chosen": -0.2223785400390625, + "rewards/margins": 1.3282341003417968, + "rewards/rejected": -1.550653076171875, + "step": 1780 + }, + { + "entropy": 1.23270263671875, + "epoch": 0.8395626300823733, + "grad_norm": 46.602272033691406, + "learning_rate": 8.963001563314227e-08, + "logits/chosen": -0.385768303897686, + "logits/rejected": 0.06807283903709165, + "logps/chosen": -484.478125, + "logps/rejected": -650.49375, + "loss": 0.5179962158203125, + "mean_token_accuracy": 0.7067975364625454, + "num_tokens": 300670752.0, + "rewards/accuracies": 0.71484375, + "rewards/chosen": -0.20882492065429686, + "rewards/margins": 1.1724075317382812, + "rewards/rejected": -1.3815521240234374, + "step": 1790 + }, + { + "entropy": 1.2193115234375, + "epoch": 0.8442529241051798, + "grad_norm": 42.331512451171875, + "learning_rate": 8.702449192287648e-08, + "logits/chosen": -0.33325068389407, + "logits/rejected": 0.12141434221741103, + "logps/chosen": -460.55625, + "logps/rejected": -658.896875, + "loss": 0.490618896484375, + "mean_token_accuracy": 0.715523487702012, + "num_tokens": 302408053.0, + "rewards/accuracies": 0.73046875, + "rewards/chosen": -0.14317150115966798, + "rewards/margins": 1.2409027099609375, + "rewards/rejected": -1.383633804321289, + "step": 1800 + }, + { + "entropy": 1.2162841796875, + "epoch": 0.8489432181279863, + "grad_norm": 38.72737503051758, + "learning_rate": 8.441896821261072e-08, + "logits/chosen": -0.3193038072324029, + "logits/rejected": 0.09296690443437472, + "logps/chosen": -473.86875, + "logps/rejected": -688.78125, + "loss": 0.5184173583984375, + "mean_token_accuracy": 0.7158959347754716, + "num_tokens": 304140722.0, + "rewards/accuracies": 0.7109375, + "rewards/chosen": -0.2360006332397461, + "rewards/margins": 1.1482696533203125, + "rewards/rejected": -1.3845916748046876, + "step": 1810 + }, + { + "entropy": 1.21527099609375, + "epoch": 0.853633512150793, + "grad_norm": 45.180023193359375, + "learning_rate": 8.181344450234496e-08, + "logits/chosen": -0.3929161294335359, + "logits/rejected": 0.15350176675456656, + "logps/chosen": -483.09375, + "logps/rejected": -646.66875, + "loss": 0.4897247314453125, + "mean_token_accuracy": 0.7121978150680661, + "num_tokens": 305822256.0, + "rewards/accuracies": 0.72265625, + "rewards/chosen": -0.18443603515625, + "rewards/margins": 1.2927734375, + "rewards/rejected": -1.47667236328125, + "step": 1820 + }, + { + "entropy": 1.1842041015625, + "epoch": 0.8583238061735995, + "grad_norm": 48.51628112792969, + "learning_rate": 7.92079207920792e-08, + "logits/chosen": -0.3951759013914627, + "logits/rejected": -0.06797198136206242, + "logps/chosen": -448.1, + "logps/rejected": -632.9, + "loss": 0.5045974731445313, + "mean_token_accuracy": 0.7215316914021969, + "num_tokens": 307531146.0, + "rewards/accuracies": 0.7125, + "rewards/chosen": -0.2178356170654297, + "rewards/margins": 1.1380462646484375, + "rewards/rejected": -1.356158447265625, + "step": 1830 + }, + { + "entropy": 1.26640625, + "epoch": 0.863014100196406, + "grad_norm": 39.41709899902344, + "learning_rate": 7.660239708181344e-08, + "logits/chosen": -0.38910823828454666, + "logits/rejected": 0.13045722797837564, + "logps/chosen": -468.475, + "logps/rejected": -673.340625, + "loss": 0.47516441345214844, + "mean_token_accuracy": 0.7065185211598873, + "num_tokens": 309213627.0, + "rewards/accuracies": 0.72265625, + "rewards/chosen": -0.20412101745605468, + "rewards/margins": 1.3498138427734374, + "rewards/rejected": -1.55379638671875, + "step": 1840 + }, + { + "entropy": 1.221923828125, + "epoch": 0.8677043942192126, + "grad_norm": 47.1512565612793, + "learning_rate": 7.399687337154768e-08, + "logits/chosen": -0.3544700164600053, + "logits/rejected": 0.013244739388591794, + "logps/chosen": -472.465625, + "logps/rejected": -669.546875, + "loss": 0.5173255920410156, + "mean_token_accuracy": 0.714903998747468, + "num_tokens": 310921408.0, + "rewards/accuracies": 0.6953125, + "rewards/chosen": -0.25082285404205323, + "rewards/margins": 1.2442939758300782, + "rewards/rejected": -1.49520263671875, + "step": 1850 + }, + { + "entropy": 1.2416259765625, + "epoch": 0.8723946882420192, + "grad_norm": 55.58268356323242, + "learning_rate": 7.139134966128191e-08, + "logits/chosen": -0.42932233723869945, + "logits/rejected": 0.006661953719145919, + "logps/chosen": -485.2625, + "logps/rejected": -650.05625, + "loss": 0.5058219909667969, + "mean_token_accuracy": 0.7036891637369991, + "num_tokens": 312542177.0, + "rewards/accuracies": 0.7109375, + "rewards/chosen": -0.18791580200195312, + "rewards/margins": 1.2160125732421876, + "rewards/rejected": -1.403900146484375, + "step": 1860 + }, + { + "entropy": 1.3213134765625, + "epoch": 0.8770849822648257, + "grad_norm": 66.06620788574219, + "learning_rate": 6.878582595101615e-08, + "logits/chosen": -0.34825132475588183, + "logits/rejected": 0.0558576905787664, + "logps/chosen": -517.925, + "logps/rejected": -723.8125, + "loss": 0.5183258056640625, + "mean_token_accuracy": 0.6997947439551353, + "num_tokens": 314220483.0, + "rewards/accuracies": 0.71953125, + "rewards/chosen": -0.23440055847167968, + "rewards/margins": 1.1802268981933595, + "rewards/rejected": -1.41419677734375, + "step": 1870 + }, + { + "entropy": 1.2500732421875, + "epoch": 0.8817752762876323, + "grad_norm": 33.63868713378906, + "learning_rate": 6.618030224075039e-08, + "logits/chosen": -0.3358364325160193, + "logits/rejected": 0.1604155617960564, + "logps/chosen": -442.221875, + "logps/rejected": -656.95625, + "loss": 0.49998931884765624, + "mean_token_accuracy": 0.7081999473273755, + "num_tokens": 315871466.0, + "rewards/accuracies": 0.7265625, + "rewards/chosen": -0.18980827331542968, + "rewards/margins": 1.1712890625, + "rewards/rejected": -1.360552978515625, + "step": 1880 + }, + { + "entropy": 1.231689453125, + "epoch": 0.8864655703104388, + "grad_norm": 43.70984649658203, + "learning_rate": 6.357477853048463e-08, + "logits/chosen": -0.3156704533986772, + "logits/rejected": 0.046957729068871246, + "logps/chosen": -472.4375, + "logps/rejected": -613.6625, + "loss": 0.49947547912597656, + "mean_token_accuracy": 0.7070603730157018, + "num_tokens": 317495293.0, + "rewards/accuracies": 0.73046875, + "rewards/chosen": -0.2524730682373047, + "rewards/margins": 1.140589141845703, + "rewards/rejected": -1.3932937622070312, + "step": 1890 + }, + { + "entropy": 1.292578125, + "epoch": 0.8911558643332453, + "grad_norm": 45.049705505371094, + "learning_rate": 6.096925482021887e-08, + "logits/chosen": -0.4336513052648586, + "logits/rejected": 0.0579325140142804, + "logps/chosen": -474.175, + "logps/rejected": -707.24375, + "loss": 0.4972557067871094, + "mean_token_accuracy": 0.7035533275455237, + "num_tokens": 319170192.0, + "rewards/accuracies": 0.7078125, + "rewards/chosen": -0.2193094253540039, + "rewards/margins": 1.2396873474121093, + "rewards/rejected": -1.4588394165039062, + "step": 1900 + }, + { + "entropy": 1.204248046875, + "epoch": 0.895846158356052, + "grad_norm": 40.22047805786133, + "learning_rate": 5.836373110995309e-08, + "logits/chosen": -0.3028328678177533, + "logits/rejected": 0.12352224319010548, + "logps/chosen": -486.69375, + "logps/rejected": -669.490625, + "loss": 0.5012863159179688, + "mean_token_accuracy": 0.7151582084596158, + "num_tokens": 320877211.0, + "rewards/accuracies": 0.70625, + "rewards/chosen": -0.19364986419677735, + "rewards/margins": 1.2327484130859374, + "rewards/rejected": -1.4259002685546875, + "step": 1910 + }, + { + "entropy": 1.215087890625, + "epoch": 0.9005364523788585, + "grad_norm": 87.88335418701172, + "learning_rate": 5.575820739968733e-08, + "logits/chosen": -0.37968428679909233, + "logits/rejected": 0.042592474067907934, + "logps/chosen": -451.80625, + "logps/rejected": -589.6625, + "loss": 0.5153121948242188, + "mean_token_accuracy": 0.711966947093606, + "num_tokens": 322486844.0, + "rewards/accuracies": 0.72265625, + "rewards/chosen": -0.227374267578125, + "rewards/margins": 1.1719680786132813, + "rewards/rejected": -1.3991973876953125, + "step": 1920 + }, + { + "entropy": 1.2044921875, + "epoch": 0.905226746401665, + "grad_norm": 54.445945739746094, + "learning_rate": 5.315268368942157e-08, + "logits/chosen": -0.3291715983156379, + "logits/rejected": 0.01717249772363879, + "logps/chosen": -481.8125, + "logps/rejected": -601.60625, + "loss": 0.5008369445800781, + "mean_token_accuracy": 0.7127945670858026, + "num_tokens": 324157116.0, + "rewards/accuracies": 0.71015625, + "rewards/chosen": -0.18655776977539062, + "rewards/margins": 1.2192840576171875, + "rewards/rejected": -1.4057220458984374, + "step": 1930 + }, + { + "entropy": 1.2319091796875, + "epoch": 0.9099170404244716, + "grad_norm": 47.28034973144531, + "learning_rate": 5.054715997915581e-08, + "logits/chosen": -0.37527385347360676, + "logits/rejected": -0.05364750899153496, + "logps/chosen": -463.29375, + "logps/rejected": -625.09375, + "loss": 0.49831314086914064, + "mean_token_accuracy": 0.7146999042481184, + "num_tokens": 325827616.0, + "rewards/accuracies": 0.73671875, + "rewards/chosen": -0.1948223114013672, + "rewards/margins": 1.2142105102539062, + "rewards/rejected": -1.4088592529296875, + "step": 1940 + }, + { + "entropy": 1.28642578125, + "epoch": 0.9146073344472782, + "grad_norm": 39.064815521240234, + "learning_rate": 4.794163626889004e-08, + "logits/chosen": -0.3388733610444481, + "logits/rejected": 0.12303869993057844, + "logps/chosen": -498.6375, + "logps/rejected": -695.2, + "loss": 0.5198501586914063, + "mean_token_accuracy": 0.7057459820061922, + "num_tokens": 327526585.0, + "rewards/accuracies": 0.709375, + "rewards/chosen": -0.20029373168945314, + "rewards/margins": 1.1490142822265625, + "rewards/rejected": -1.3491470336914062, + "step": 1950 + }, + { + "entropy": 1.280712890625, + "epoch": 0.9192976284700847, + "grad_norm": 43.056724548339844, + "learning_rate": 4.533611255862428e-08, + "logits/chosen": -0.33125824228419354, + "logits/rejected": 0.06951288466580796, + "logps/chosen": -505.46875, + "logps/rejected": -757.3375, + "loss": 0.49418106079101565, + "mean_token_accuracy": 0.7125601809471845, + "num_tokens": 329344689.0, + "rewards/accuracies": 0.71875, + "rewards/chosen": -0.2517139434814453, + "rewards/margins": 1.2828144073486327, + "rewards/rejected": -1.534075927734375, + "step": 1960 + }, + { + "entropy": 1.3103759765625, + "epoch": 0.9239879224928913, + "grad_norm": 47.633358001708984, + "learning_rate": 4.273058884835852e-08, + "logits/chosen": -0.32412154662737075, + "logits/rejected": 0.14630080084089, + "logps/chosen": -507.8625, + "logps/rejected": -787.490625, + "loss": 0.49291152954101564, + "mean_token_accuracy": 0.7110428418964148, + "num_tokens": 331076977.0, + "rewards/accuracies": 0.72265625, + "rewards/chosen": -0.2450836181640625, + "rewards/margins": 1.400872802734375, + "rewards/rejected": -1.6462799072265626, + "step": 1970 + }, + { + "entropy": 1.2950927734375, + "epoch": 0.9286782165156978, + "grad_norm": 40.30205154418945, + "learning_rate": 4.0125065138092755e-08, + "logits/chosen": -0.3315387981538188, + "logits/rejected": 0.1416249235004452, + "logps/chosen": -485.65, + "logps/rejected": -699.46875, + "loss": 0.4833381652832031, + "mean_token_accuracy": 0.7057482942938804, + "num_tokens": 332736113.0, + "rewards/accuracies": 0.72890625, + "rewards/chosen": -0.2063751220703125, + "rewards/margins": 1.2729263305664062, + "rewards/rejected": -1.4789794921875, + "step": 1980 + }, + { + "entropy": 1.30361328125, + "epoch": 0.9333685105385043, + "grad_norm": 51.24271774291992, + "learning_rate": 3.751954142782699e-08, + "logits/chosen": -0.3614101444479402, + "logits/rejected": 0.12883004161401715, + "logps/chosen": -457.2375, + "logps/rejected": -743.24375, + "loss": 0.50074462890625, + "mean_token_accuracy": 0.7107256408780813, + "num_tokens": 334425608.0, + "rewards/accuracies": 0.72109375, + "rewards/chosen": -0.21077451705932618, + "rewards/margins": 1.164959716796875, + "rewards/rejected": -1.3759368896484374, + "step": 1990 + }, + { + "entropy": 1.240966796875, + "epoch": 0.938058804561311, + "grad_norm": 40.48645782470703, + "learning_rate": 3.491401771756123e-08, + "logits/chosen": -0.42722963460593266, + "logits/rejected": 0.08344267947642378, + "logps/chosen": -463.0375, + "logps/rejected": -612.61875, + "loss": 0.4973747253417969, + "mean_token_accuracy": 0.7031021565198898, + "num_tokens": 336040370.0, + "rewards/accuracies": 0.7234375, + "rewards/chosen": -0.19219436645507812, + "rewards/margins": 1.2013168334960938, + "rewards/rejected": -1.3940261840820312, + "step": 2000 + }, + { + "entropy": 1.2700927734375, + "epoch": 0.9427490985841175, + "grad_norm": 43.718074798583984, + "learning_rate": 3.230849400729546e-08, + "logits/chosen": -0.33091222728590197, + "logits/rejected": 0.18359424973817012, + "logps/chosen": -492.859375, + "logps/rejected": -667.65, + "loss": 0.4870887756347656, + "mean_token_accuracy": 0.7044433875009417, + "num_tokens": 337738884.0, + "rewards/accuracies": 0.71953125, + "rewards/chosen": -0.20358028411865234, + "rewards/margins": 1.360540771484375, + "rewards/rejected": -1.56429443359375, + "step": 2010 + }, + { + "entropy": 1.25888671875, + "epoch": 0.947439392606924, + "grad_norm": 48.371150970458984, + "learning_rate": 2.97029702970297e-08, + "logits/chosen": -0.3781629436293919, + "logits/rejected": 0.05601582152754293, + "logps/chosen": -474.40625, + "logps/rejected": -704.209375, + "loss": 0.5054107666015625, + "mean_token_accuracy": 0.7089995898306369, + "num_tokens": 339374987.0, + "rewards/accuracies": 0.7078125, + "rewards/chosen": -0.2509334564208984, + "rewards/margins": 1.4051788330078125, + "rewards/rejected": -1.65648193359375, + "step": 2020 + }, + { + "entropy": 1.2791015625, + "epoch": 0.9521296866297306, + "grad_norm": 45.81821823120117, + "learning_rate": 2.709744658676394e-08, + "logits/chosen": -0.3524803996629535, + "logits/rejected": 0.16845931554752744, + "logps/chosen": -458.95625, + "logps/rejected": -714.40625, + "loss": 0.5151969909667968, + "mean_token_accuracy": 0.70955845490098, + "num_tokens": 341081547.0, + "rewards/accuracies": 0.68984375, + "rewards/chosen": -0.19017089605331422, + "rewards/margins": 1.173492431640625, + "rewards/rejected": -1.364105224609375, + "step": 2030 + }, + { + "entropy": 1.2907958984375, + "epoch": 0.9568199806525371, + "grad_norm": 41.41585922241211, + "learning_rate": 2.4491922876498173e-08, + "logits/chosen": -0.4542369755451925, + "logits/rejected": -0.058119418372802836, + "logps/chosen": -481.484375, + "logps/rejected": -672.94375, + "loss": 0.49808197021484374, + "mean_token_accuracy": 0.7006991088390351, + "num_tokens": 342748476.0, + "rewards/accuracies": 0.73359375, + "rewards/chosen": -0.15017204284667968, + "rewards/margins": 1.2583206176757813, + "rewards/rejected": -1.4087432861328124, + "step": 2040 + }, + { + "entropy": 1.2560546875, + "epoch": 0.9615102746753437, + "grad_norm": 47.81901931762695, + "learning_rate": 2.1886399166232413e-08, + "logits/chosen": -0.35172637393118933, + "logits/rejected": 0.08463381700840975, + "logps/chosen": -490.5375, + "logps/rejected": -701.64375, + "loss": 0.4924041748046875, + "mean_token_accuracy": 0.7111782263964415, + "num_tokens": 344461836.0, + "rewards/accuracies": 0.7203125, + "rewards/chosen": -0.19839134216308593, + "rewards/margins": 1.3799209594726562, + "rewards/rejected": -1.57762451171875, + "step": 2050 + }, + { + "entropy": 1.2580810546875, + "epoch": 0.9662005686981503, + "grad_norm": 42.42911911010742, + "learning_rate": 1.928087545596665e-08, + "logits/chosen": -0.3737677612104985, + "logits/rejected": 0.0983303444858018, + "logps/chosen": -433.90625, + "logps/rejected": -661.34375, + "loss": 0.484228515625, + "mean_token_accuracy": 0.7108163654804229, + "num_tokens": 346122958.0, + "rewards/accuracies": 0.71875, + "rewards/chosen": -0.13962364196777344, + "rewards/margins": 1.2540679931640626, + "rewards/rejected": -1.3932327270507812, + "step": 2060 + }, + { + "entropy": 1.2074462890625, + "epoch": 0.9708908627209568, + "grad_norm": 47.9688720703125, + "learning_rate": 1.6675351745700885e-08, + "logits/chosen": -0.3346860115742126, + "logits/rejected": 0.13174351469037554, + "logps/chosen": -463.53125, + "logps/rejected": -660.0875, + "loss": 0.5093505859375, + "mean_token_accuracy": 0.7194312229752541, + "num_tokens": 347810695.0, + "rewards/accuracies": 0.6984375, + "rewards/chosen": -0.22800559997558595, + "rewards/margins": 1.2211044311523438, + "rewards/rejected": -1.4488922119140626, + "step": 2070 + }, + { + "entropy": 1.229345703125, + "epoch": 0.9755811567437633, + "grad_norm": 41.37725830078125, + "learning_rate": 1.4069828035435123e-08, + "logits/chosen": -0.3518116668610031, + "logits/rejected": 0.1489972835310932, + "logps/chosen": -456.98125, + "logps/rejected": -670.26875, + "loss": 0.5096603393554687, + "mean_token_accuracy": 0.71624949388206, + "num_tokens": 349502012.0, + "rewards/accuracies": 0.70546875, + "rewards/chosen": -0.2621330261230469, + "rewards/margins": 1.2914962768554688, + "rewards/rejected": -1.5537750244140625, + "step": 2080 + }, + { + "entropy": 1.2279052734375, + "epoch": 0.98027145076657, + "grad_norm": 51.86234664916992, + "learning_rate": 1.1464304325169358e-08, + "logits/chosen": -0.3000658016784428, + "logits/rejected": 0.17493088714280344, + "logps/chosen": -478.8, + "logps/rejected": -717.7, + "loss": 0.505853271484375, + "mean_token_accuracy": 0.7180974125862122, + "num_tokens": 351224924.0, + "rewards/accuracies": 0.7078125, + "rewards/chosen": -0.2635894775390625, + "rewards/margins": 1.29547119140625, + "rewards/rejected": -1.55958251953125, + "step": 2090 + }, + { + "entropy": 1.2322509765625, + "epoch": 0.9849617447893765, + "grad_norm": 51.60932159423828, + "learning_rate": 8.858780614903594e-09, + "logits/chosen": -0.35039016367491677, + "logits/rejected": 0.10474107382362959, + "logps/chosen": -484.2, + "logps/rejected": -661.559375, + "loss": 0.5166488647460937, + "mean_token_accuracy": 0.7098216339945793, + "num_tokens": 352931771.0, + "rewards/accuracies": 0.68828125, + "rewards/chosen": -0.22312393188476562, + "rewards/margins": 1.132904052734375, + "rewards/rejected": -1.35577392578125, + "step": 2100 + }, + { + "entropy": 1.155126953125, + "epoch": 0.989652038812183, + "grad_norm": 46.09328842163086, + "learning_rate": 6.253256904637832e-09, + "logits/chosen": -0.3290858068608348, + "logits/rejected": -0.02093571722635537, + "logps/chosen": -485.83125, + "logps/rejected": -543.66875, + "loss": 0.5270843505859375, + "mean_token_accuracy": 0.713743943348527, + "num_tokens": 354621334.0, + "rewards/accuracies": 0.7046875, + "rewards/chosen": -0.19669809341430664, + "rewards/margins": 1.0803466796875, + "rewards/rejected": -1.276715087890625, + "step": 2110 + }, + { + "entropy": 1.2471435546875, + "epoch": 0.9943423328349896, + "grad_norm": 65.90406036376953, + "learning_rate": 3.6477331943720682e-09, + "logits/chosen": -0.20736095787263226, + "logits/rejected": 0.1461316692738893, + "logps/chosen": -506.425, + "logps/rejected": -653.98125, + "loss": 0.5250595092773438, + "mean_token_accuracy": 0.7124230587854982, + "num_tokens": 356312751.0, + "rewards/accuracies": 0.71640625, + "rewards/chosen": -0.22593250274658203, + "rewards/margins": 1.2058555603027343, + "rewards/rejected": -1.4319067001342773, + "step": 2120 + }, + { + "entropy": 1.2607177734375, + "epoch": 0.9990326268577961, + "grad_norm": 44.84026336669922, + "learning_rate": 1.0422094841063053e-09, + "logits/chosen": -0.3921418517965115, + "logits/rejected": -0.05872409453138806, + "logps/chosen": -464.925, + "logps/rejected": -629.39375, + "loss": 0.49729080200195314, + "mean_token_accuracy": 0.7027096889913083, + "num_tokens": 357948612.0, + "rewards/accuracies": 0.72421875, + "rewards/chosen": -0.2333770751953125, + "rewards/margins": 1.1683845520019531, + "rewards/rejected": -1.4017159461975097, + "step": 2130 + }, + { + "entropy": 1.178030303030303, + "epoch": 1.0, + "logits/chosen": -0.30206525592803823, + "logits/rejected": -0.0929858402560999, + "logps/chosen": -472.09090909090907, + "logps/rejected": -557.5757575757576, + "mean_token_accuracy": 0.7097648002884605, + "num_tokens": 358277178.0, + "rewards/accuracies": 0.7234848484848485, + "rewards/chosen": -0.24461780894886365, + "rewards/margins": 1.0697687322443181, + "rewards/rejected": -1.3149266098484849, + "step": 2133, + "total_flos": 4.975817267087409e+18, + "train_loss": 0.527936527665322, + "train_runtime": 22045.0555, + "train_samples_per_second": 12.379, + "train_steps_per_second": 0.097 + } + ], + "logging_steps": 10, + "max_steps": 2133, + "num_input_tokens_seen": 0, + "num_train_epochs": 1, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": true + }, + "attributes": {} + } + }, + "total_flos": 4.975817267087409e+18, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +} diff --git a/training_args.bin b/training_args.bin new file mode 100644 index 0000000..2b7c05f --- /dev/null +++ b/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8828eabdb4b872c3014098f2096552dbfaf34429265b98e632bffaa738211407 +size 7377