commit 2a3154f895342aa934c8d1a4e0b6316f89dfe3ff Author: ModelHub XC Date: Mon Jul 27 07:41:10 2026 +0800 初始化项目，由ModelHub XC社区提供模型 Model: Wenboz/zephyr-7b-dpo-full Source: Original Platform diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..52373fe --- /dev/null +++ b/.gitattributes @@ -0,0 +1,36 @@ +*.7z filter=lfs diff=lfs merge=lfs -text +*.arrow filter=lfs diff=lfs merge=lfs -text +*.bin filter=lfs diff=lfs merge=lfs -text +*.bz2 filter=lfs diff=lfs merge=lfs -text +*.ckpt filter=lfs diff=lfs merge=lfs -text +*.ftz filter=lfs diff=lfs merge=lfs -text +*.gz filter=lfs diff=lfs merge=lfs -text +*.h5 filter=lfs diff=lfs merge=lfs -text +*.joblib filter=lfs diff=lfs merge=lfs -text +*.lfs.* filter=lfs diff=lfs merge=lfs -text +*.mlmodel filter=lfs diff=lfs merge=lfs -text +*.model filter=lfs diff=lfs merge=lfs -text +*.msgpack filter=lfs diff=lfs merge=lfs -text +*.npy filter=lfs diff=lfs merge=lfs -text +*.npz filter=lfs diff=lfs merge=lfs -text +*.onnx filter=lfs diff=lfs merge=lfs -text +*.ot filter=lfs diff=lfs merge=lfs -text +*.parquet filter=lfs diff=lfs merge=lfs -text +*.pb filter=lfs diff=lfs merge=lfs -text +*.pickle filter=lfs diff=lfs merge=lfs -text +*.pkl filter=lfs diff=lfs merge=lfs -text +*.pt filter=lfs diff=lfs merge=lfs -text +*.pth filter=lfs diff=lfs merge=lfs -text +*.rar filter=lfs diff=lfs merge=lfs -text +*.safetensors filter=lfs diff=lfs merge=lfs -text +saved_model/**/* filter=lfs diff=lfs merge=lfs -text +*.tar.* filter=lfs diff=lfs merge=lfs -text +*.tar filter=lfs diff=lfs merge=lfs -text +*.tflite filter=lfs diff=lfs merge=lfs -text +*.tgz filter=lfs diff=lfs merge=lfs -text +*.wasm filter=lfs diff=lfs merge=lfs -text +*.xz filter=lfs diff=lfs merge=lfs -text +*.zip filter=lfs diff=lfs merge=lfs -text +*.zst filter=lfs diff=lfs merge=lfs -text +*tfevents* filter=lfs diff=lfs merge=lfs -text +tokenizer.json filter=lfs diff=lfs merge=lfs -text diff --git a/README.md b/README.md new file mode 100644 index 0000000..fcdb4f7 --- /dev/null +++ b/README.md @@ -0,0 +1,74 @@ +--- +library_name: transformers +license: apache-2.0 +base_model: alignment-handbook/zephyr-7b-sft-full +tags: +- trl +- dpo +- generated_from_trainer +model-index: +- name: zephyr-7b-dpo-full + results: [] +--- + + + +# zephyr-7b-dpo-full + +This model is a fine-tuned version of [alignment-handbook/zephyr-7b-sft-full](https://huggingface.co/alignment-handbook/zephyr-7b-sft-full) on the None dataset. +It achieves the following results on the evaluation set: +- Loss: 1.2016 +- Rewards/chosen: -5.9224 +- Rewards/rejected: -7.8036 +- Rewards/accuracies: 0.7380 +- Rewards/margins: 1.8812 +- Logps/rejected: -3.9018 +- Logps/chosen: -2.9612 +- Logits/rejected: -1.8800 +- Logits/chosen: -1.8679 + +## Model description + +More information needed + +## Intended uses & limitations + +More information needed + +## Training and evaluation data + +More information needed + +## Training procedure + +### Training hyperparameters + +The following hyperparameters were used during training: +- learning_rate: 5e-07 +- train_batch_size: 2 +- eval_batch_size: 4 +- seed: 42 +- distributed_type: multi-GPU +- num_devices: 4 +- gradient_accumulation_steps: 16 +- total_train_batch_size: 128 +- total_eval_batch_size: 16 +- optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08 +- lr_scheduler_type: cosine +- lr_scheduler_warmup_ratio: 0.1 +- num_epochs: 1 + +### Training results + +| Training Loss | Epoch | Step | Validation Loss | Rewards/chosen | Rewards/rejected | Rewards/accuracies | Rewards/margins | Logps/rejected | Logps/chosen | Logits/rejected | Logits/chosen | +|:-------------:|:------:|:----:|:---------------:|:--------------:|:----------------:|:------------------:|:---------------:|:--------------:|:------------:|:---------------:|:-------------:| +| 1.2816 | 0.8375 | 400 | 1.2016 | -5.9224 | -7.8036 | 0.7380 | 1.8812 | -3.9018 | -2.9612 | -1.8800 | -1.8679 | + + +### Framework versions + +- Transformers 4.45.2 +- Pytorch 2.4.0+cu121 +- Datasets 2.14.6 +- Tokenizers 0.20.1 diff --git a/all_results.json b/all_results.json new file mode 100644 index 0000000..99a9ddd --- /dev/null +++ b/all_results.json @@ -0,0 +1,9 @@ +{ + "epoch": 0.998691442030882, + "total_flos": 0.0, + "train_loss": 1.3635074789918944, + "train_runtime": 22192.2901, + "train_samples": 61135, + "train_samples_per_second": 2.755, + "train_steps_per_second": 0.021 +} \ No newline at end of file diff --git a/config.json b/config.json new file mode 100644 index 0000000..466a21b --- /dev/null +++ b/config.json @@ -0,0 +1,27 @@ +{ + "_name_or_path": "alignment-handbook/zephyr-7b-sft-full", + "architectures": [ + "MistralForCausalLM" + ], + "attention_dropout": 0.0, + "bos_token_id": 1, + "eos_token_id": 2, + "head_dim": 128, + "hidden_act": "silu", + "hidden_size": 4096, + "initializer_range": 0.02, + "intermediate_size": 14336, + "max_position_embeddings": 32768, + "model_type": "mistral", + "num_attention_heads": 32, + "num_hidden_layers": 32, + "num_key_value_heads": 8, + "rms_norm_eps": 1e-05, + "rope_theta": 10000.0, + "sliding_window": 4096, + "tie_word_embeddings": false, + "torch_dtype": "bfloat16", + "transformers_version": "4.45.2", + "use_cache": false, + "vocab_size": 32000 +} diff --git a/eval_results.json b/eval_results.json new file mode 100644 index 0000000..4b7a429 --- /dev/null +++ b/eval_results.json @@ -0,0 +1,16 @@ +{ + "epoch": 0.9997382884061764, + "eval_logits/chosen": -0.6658732891082764, + "eval_logits/rejected": -0.6216869950294495, + "eval_logps/chosen": -282.2928161621094, + "eval_logps/rejected": -265.7748107910156, + "eval_loss": 0.9743963479995728, + "eval_rewards/accuracies": 0.6919999718666077, + "eval_rewards/chosen": -1.00694739818573, + "eval_rewards/margins": 0.7492353320121765, + "eval_rewards/rejected": -1.7561826705932617, + "eval_runtime": 140.7369, + "eval_samples": 2000, + "eval_samples_per_second": 14.211, + "eval_steps_per_second": 1.776 +} \ No newline at end of file diff --git a/generation_config.json b/generation_config.json new file mode 100644 index 0000000..f784d96 --- /dev/null +++ b/generation_config.json @@ -0,0 +1,6 @@ +{ + "_from_model_config": true, + "bos_token_id": 1, + "eos_token_id": 2, + "transformers_version": "4.45.2" +} diff --git a/model-00001-of-00002.safetensors b/model-00001-of-00002.safetensors new file mode 100644 index 0000000..bf39611 --- /dev/null +++ b/model-00001-of-00002.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6be44a051546c0da75f26710476b6b80ceb713a277166714f465ac79b5595464 +size 4965799096 diff --git a/model-00001-of-00003.safetensors b/model-00001-of-00003.safetensors new file mode 100644 index 0000000..d999f0f --- /dev/null +++ b/model-00001-of-00003.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:667def6b765b4e1223b8dfdbc010bcdabd0a328d31ffc320c19dc4e6fdff55b7 +size 4943162336 diff --git a/model-00002-of-00002.safetensors b/model-00002-of-00002.safetensors new file mode 100644 index 0000000..08bc079 --- /dev/null +++ b/model-00002-of-00002.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:254f5caa33755de1944026fe7403c1cdb5d8db931b5f3fabc5d476a7d31f4123 +size 2247734992 diff --git a/model-00002-of-00003.safetensors b/model-00002-of-00003.safetensors new file mode 100644 index 0000000..8a27177 --- /dev/null +++ b/model-00002-of-00003.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:11a5c5ec9b3d4232ea443e7a93611a1c10c92eb3b6e82b90c13515d01137d414 +size 4999819336 diff --git a/model-00003-of-00003.safetensors b/model-00003-of-00003.safetensors new file mode 100644 index 0000000..9d0a275 --- /dev/null +++ b/model-00003-of-00003.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a318812490ff6f98f6f4a1186a176edd9b1113f5df0e485a05dd2b6fab50b58c +size 4540516344 diff --git a/model.safetensors b/model.safetensors new file mode 100644 index 0000000..076a0f8 --- /dev/null +++ b/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:67ef079b651cce11aab53846f772e0a7ccc2d7af6dbefd70ea8a36d072d5324a +size 4943274328 diff --git a/model.safetensors.index.json b/model.safetensors.index.json new file mode 100644 index 0000000..b349bc0 --- /dev/null +++ b/model.safetensors.index.json @@ -0,0 +1,298 @@ +{ + "metadata": { + "total_size": 14483464192 + }, + "weight_map": { + "lm_head.weight": "model-00003-of-00003.safetensors", + "model.embed_tokens.weight": "model-00001-of-00003.safetensors", + "model.layers.0.input_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.0.mlp.down_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.0.mlp.up_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.1.input_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.1.mlp.down_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.1.mlp.up_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.10.input_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.10.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.10.mlp.gate_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.10.mlp.up_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.10.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.10.self_attn.k_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.10.self_attn.o_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.10.self_attn.q_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.10.self_attn.v_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.11.input_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.11.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.11.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.11.mlp.up_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.11.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.11.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.11.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.11.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.11.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.12.input_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.12.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.12.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.12.mlp.up_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.12.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.12.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.12.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.12.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.12.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.13.input_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.13.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.13.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.13.mlp.up_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.13.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.13.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.13.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.13.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.13.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.14.input_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.14.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.14.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.14.mlp.up_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.14.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.14.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.14.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.14.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.14.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.15.input_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.15.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.15.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.15.mlp.up_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.15.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.15.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.15.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.15.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.15.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.16.input_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.16.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.16.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.16.mlp.up_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.16.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.16.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.16.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.16.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.16.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.17.input_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.17.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.17.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.17.mlp.up_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.17.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.17.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.17.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.17.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.17.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.18.input_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.18.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.18.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.18.mlp.up_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.18.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.18.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.18.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.18.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.18.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.19.input_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.19.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.19.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.19.mlp.up_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.19.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.19.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.19.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.19.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.19.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.2.input_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.2.mlp.down_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.2.mlp.up_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.20.input_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.20.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.20.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.20.mlp.up_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.20.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.20.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.20.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.20.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.20.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.21.input_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.21.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.21.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.21.mlp.up_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.21.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.21.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.21.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.21.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.21.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.22.input_layernorm.weight": "model-00003-of-00003.safetensors", + "model.layers.22.mlp.down_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.22.mlp.gate_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.22.mlp.up_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.22.post_attention_layernorm.weight": "model-00003-of-00003.safetensors", + "model.layers.22.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.22.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.22.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.22.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.23.input_layernorm.weight": "model-00003-of-00003.safetensors", + "model.layers.23.mlp.down_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.23.mlp.gate_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.23.mlp.up_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.23.post_attention_layernorm.weight": "model-00003-of-00003.safetensors", + "model.layers.23.self_attn.k_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.23.self_attn.o_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.23.self_attn.q_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.23.self_attn.v_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.24.input_layernorm.weight": "model-00003-of-00003.safetensors", + "model.layers.24.mlp.down_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.24.mlp.gate_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.24.mlp.up_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.24.post_attention_layernorm.weight": "model-00003-of-00003.safetensors", + "model.layers.24.self_attn.k_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.24.self_attn.o_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.24.self_attn.q_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.24.self_attn.v_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.25.input_layernorm.weight": "model-00003-of-00003.safetensors", + "model.layers.25.mlp.down_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.25.mlp.gate_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.25.mlp.up_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.25.post_attention_layernorm.weight": "model-00003-of-00003.safetensors", + "model.layers.25.self_attn.k_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.25.self_attn.o_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.25.self_attn.q_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.25.self_attn.v_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.26.input_layernorm.weight": "model-00003-of-00003.safetensors", + "model.layers.26.mlp.down_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.26.mlp.gate_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.26.mlp.up_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.26.post_attention_layernorm.weight": "model-00003-of-00003.safetensors", + "model.layers.26.self_attn.k_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.26.self_attn.o_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.26.self_attn.q_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.26.self_attn.v_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.27.input_layernorm.weight": "model-00003-of-00003.safetensors", + "model.layers.27.mlp.down_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.27.mlp.gate_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.27.mlp.up_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.27.post_attention_layernorm.weight": "model-00003-of-00003.safetensors", + "model.layers.27.self_attn.k_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.27.self_attn.o_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.27.self_attn.q_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.27.self_attn.v_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.28.input_layernorm.weight": "model-00003-of-00003.safetensors", + "model.layers.28.mlp.down_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.28.mlp.gate_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.28.mlp.up_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.28.post_attention_layernorm.weight": "model-00003-of-00003.safetensors", + "model.layers.28.self_attn.k_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.28.self_attn.o_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.28.self_attn.q_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.28.self_attn.v_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.29.input_layernorm.weight": "model-00003-of-00003.safetensors", + "model.layers.29.mlp.down_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.29.mlp.gate_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.29.mlp.up_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.29.post_attention_layernorm.weight": "model-00003-of-00003.safetensors", + "model.layers.29.self_attn.k_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.29.self_attn.o_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.29.self_attn.q_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.29.self_attn.v_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.3.input_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.3.mlp.down_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.3.mlp.up_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.30.input_layernorm.weight": "model-00003-of-00003.safetensors", + "model.layers.30.mlp.down_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.30.mlp.gate_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.30.mlp.up_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.30.post_attention_layernorm.weight": "model-00003-of-00003.safetensors", + "model.layers.30.self_attn.k_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.30.self_attn.o_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.30.self_attn.q_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.30.self_attn.v_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.31.input_layernorm.weight": "model-00003-of-00003.safetensors", + "model.layers.31.mlp.down_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.31.mlp.gate_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.31.mlp.up_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.31.post_attention_layernorm.weight": "model-00003-of-00003.safetensors", + "model.layers.31.self_attn.k_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.31.self_attn.o_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.31.self_attn.q_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.31.self_attn.v_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.4.input_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.4.mlp.down_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.4.mlp.up_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.5.input_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.5.mlp.down_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.5.mlp.up_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.6.input_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.6.mlp.down_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.6.mlp.gate_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.6.mlp.up_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.7.input_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.7.mlp.down_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.7.mlp.gate_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.7.mlp.up_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.7.post_attention_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.7.self_attn.k_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.7.self_attn.o_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.7.self_attn.q_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.7.self_attn.v_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.8.input_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.8.mlp.down_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.8.mlp.gate_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.8.mlp.up_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.8.post_attention_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.8.self_attn.k_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.8.self_attn.o_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.8.self_attn.q_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.8.self_attn.v_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.9.input_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.9.mlp.down_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.9.mlp.gate_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.9.mlp.up_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.9.post_attention_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.9.self_attn.k_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.9.self_attn.o_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.9.self_attn.q_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.9.self_attn.v_proj.weight": "model-00001-of-00003.safetensors", + "model.norm.weight": "model-00003-of-00003.safetensors" + } +} diff --git a/special_tokens_map.json b/special_tokens_map.json new file mode 100644 index 0000000..492d4b2 --- /dev/null +++ b/special_tokens_map.json @@ -0,0 +1,30 @@ +{ + "bos_token": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "eos_token": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "pad_token": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "unk_token": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + } +} diff --git a/tokenizer.json b/tokenizer.json new file mode 100644 index 0000000..54bbfe3 --- /dev/null +++ b/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1d5b9634c23bdd4540f633d327120e1fa4b57a2a723a56d7a92debfc4d15c061 +size 3505751 diff --git a/tokenizer_config.json b/tokenizer_config.json new file mode 100644 index 0000000..68dfc38 --- /dev/null +++ b/tokenizer_config.json @@ -0,0 +1,44 @@ +{ + "add_bos_token": true, + "add_eos_token": false, + "add_prefix_space": null, + "added_tokens_decoder": { + "0": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "1": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "2": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + } + }, + "additional_special_tokens": [], + "bos_token": "", + "chat_template": "{% for message in messages %}\n{% if message['role'] == 'user' %}\n{{ '<|user|>\n' + message['content'] + eos_token }}\n{% elif message['role'] == 'system' %}\n{{ '<|system|>\n' + message['content'] + eos_token }}\n{% elif message['role'] == 'assistant' %}\n{{ '<|assistant|>\n' + message['content'] + eos_token }}\n{% endif %}\n{% if loop.last and add_generation_prompt %}\n{{ '<|assistant|>' }}\n{% endif %}\n{% endfor %}", + "clean_up_tokenization_spaces": false, + "eos_token": "", + "legacy": true, + "model_max_length": 2048, + "pad_token": "", + "sp_model_kwargs": {}, + "spaces_between_special_tokens": false, + "tokenizer_class": "LlamaTokenizer", + "unk_token": "", + "use_default_system_prompt": false +} diff --git a/train_results.json b/train_results.json new file mode 100644 index 0000000..99a9ddd --- /dev/null +++ b/train_results.json @@ -0,0 +1,9 @@ +{ + "epoch": 0.998691442030882, + "total_flos": 0.0, + "train_loss": 1.3635074789918944, + "train_runtime": 22192.2901, + "train_samples": 61135, + "train_samples_per_second": 2.755, + "train_steps_per_second": 0.021 +} \ No newline at end of file diff --git a/trainer_state.json b/trainer_state.json new file mode 100644 index 0000000..40a2b19 --- /dev/null +++ b/trainer_state.json @@ -0,0 +1,1498 @@ +{ + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 0.998691442030882, + "eval_steps": 400, + "global_step": 477, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.002093692750588851, + "grad_norm": 23.60069859438079, + "learning_rate": 1.0416666666666666e-08, + "logits/chosen": -2.8280014991760254, + "logits/rejected": -2.8466408252716064, + "logps/chosen": -1.1081000566482544, + "logps/rejected": -1.146370530128479, + "loss": 1.825, + "rewards/accuracies": 0.46875, + "rewards/chosen": -2.216200113296509, + "rewards/margins": 0.0765407383441925, + "rewards/rejected": -2.292741060256958, + "step": 1 + }, + { + "epoch": 0.010468463752944255, + "grad_norm": 20.150796004914934, + "learning_rate": 5.208333333333333e-08, + "logits/chosen": -2.8811752796173096, + "logits/rejected": -2.847626209259033, + "logps/chosen": -1.0211418867111206, + "logps/rejected": -1.080704689025879, + "loss": 1.7105, + "rewards/accuracies": 0.5390625, + "rewards/chosen": -2.042283773422241, + "rewards/margins": 0.11912553012371063, + "rewards/rejected": -2.161409378051758, + "step": 5 + }, + { + "epoch": 0.02093692750588851, + "grad_norm": 19.30396074137424, + "learning_rate": 1.0416666666666667e-07, + "logits/chosen": -2.992589235305786, + "logits/rejected": -2.9415202140808105, + "logps/chosen": -1.0235936641693115, + "logps/rejected": -1.1658079624176025, + "loss": 1.7413, + "rewards/accuracies": 0.581250011920929, + "rewards/chosen": -2.047187328338623, + "rewards/margins": 0.2844284772872925, + "rewards/rejected": -2.331615924835205, + "step": 10 + }, + { + "epoch": 0.031405391258832765, + "grad_norm": 22.37404641220351, + "learning_rate": 1.5624999999999999e-07, + "logits/chosen": -2.9186646938323975, + "logits/rejected": -2.905055522918701, + "logps/chosen": -1.0526732206344604, + "logps/rejected": -1.1032135486602783, + "loss": 1.7229, + "rewards/accuracies": 0.5375000238418579, + "rewards/chosen": -2.105346441268921, + "rewards/margins": 0.10108034312725067, + "rewards/rejected": -2.2064270973205566, + "step": 15 + }, + { + "epoch": 0.04187385501177702, + "grad_norm": 18.24180275182366, + "learning_rate": 2.0833333333333333e-07, + "logits/chosen": -2.9416229724884033, + "logits/rejected": -2.890882968902588, + "logps/chosen": -1.0062425136566162, + "logps/rejected": -1.1030701398849487, + "loss": 1.6956, + "rewards/accuracies": 0.550000011920929, + "rewards/chosen": -2.0124850273132324, + "rewards/margins": 0.19365549087524414, + "rewards/rejected": -2.2061402797698975, + "step": 20 + }, + { + "epoch": 0.05234231876472128, + "grad_norm": 17.74156497951028, + "learning_rate": 2.604166666666667e-07, + "logits/chosen": -2.9460272789001465, + "logits/rejected": -2.882147789001465, + "logps/chosen": -0.9283784031867981, + "logps/rejected": -1.1168667078018188, + "loss": 1.664, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -1.8567568063735962, + "rewards/margins": 0.37697672843933105, + "rewards/rejected": -2.2337334156036377, + "step": 25 + }, + { + "epoch": 0.06281078251766553, + "grad_norm": 17.414699170293087, + "learning_rate": 3.1249999999999997e-07, + "logits/chosen": -2.919449806213379, + "logits/rejected": -2.9000954627990723, + "logps/chosen": -0.9085705876350403, + "logps/rejected": -0.9413141012191772, + "loss": 1.6764, + "rewards/accuracies": 0.550000011920929, + "rewards/chosen": -1.8171411752700806, + "rewards/margins": 0.06548723578453064, + "rewards/rejected": -1.8826282024383545, + "step": 30 + }, + { + "epoch": 0.07327924627060979, + "grad_norm": 18.306181256584207, + "learning_rate": 3.645833333333333e-07, + "logits/chosen": -2.929173469543457, + "logits/rejected": -2.8954787254333496, + "logps/chosen": -0.9194827079772949, + "logps/rejected": -1.1145708560943604, + "loss": 1.6503, + "rewards/accuracies": 0.606249988079071, + "rewards/chosen": -1.8389654159545898, + "rewards/margins": 0.39017611742019653, + "rewards/rejected": -2.2291417121887207, + "step": 35 + }, + { + "epoch": 0.08374771002355404, + "grad_norm": 17.72310110786498, + "learning_rate": 4.1666666666666667e-07, + "logits/chosen": -2.920375347137451, + "logits/rejected": -2.878530979156494, + "logps/chosen": -0.9253458976745605, + "logps/rejected": -1.0473893880844116, + "loss": 1.6223, + "rewards/accuracies": 0.581250011920929, + "rewards/chosen": -1.850691795349121, + "rewards/margins": 0.2440868616104126, + "rewards/rejected": -2.0947787761688232, + "step": 40 + }, + { + "epoch": 0.0942161737764983, + "grad_norm": 16.885070690793345, + "learning_rate": 4.6874999999999996e-07, + "logits/chosen": -2.875908851623535, + "logits/rejected": -2.8810861110687256, + "logps/chosen": -0.903559684753418, + "logps/rejected": -1.0813857316970825, + "loss": 1.5727, + "rewards/accuracies": 0.625, + "rewards/chosen": -1.807119369506836, + "rewards/margins": 0.3556520342826843, + "rewards/rejected": -2.162771463394165, + "step": 45 + }, + { + "epoch": 0.10468463752944256, + "grad_norm": 23.820902499939343, + "learning_rate": 4.999731868769026e-07, + "logits/chosen": -2.820807933807373, + "logits/rejected": -2.832883358001709, + "logps/chosen": -0.9573640823364258, + "logps/rejected": -1.2130459547042847, + "loss": 1.6274, + "rewards/accuracies": 0.637499988079071, + "rewards/chosen": -1.9147281646728516, + "rewards/margins": 0.5113638043403625, + "rewards/rejected": -2.4260919094085693, + "step": 50 + }, + { + "epoch": 0.11515310128238682, + "grad_norm": 24.790862979912625, + "learning_rate": 4.996716052911017e-07, + "logits/chosen": -2.9123668670654297, + "logits/rejected": -2.9268505573272705, + "logps/chosen": -1.0739004611968994, + "logps/rejected": -1.1845686435699463, + "loss": 1.6259, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -2.147800922393799, + "rewards/margins": 0.22133660316467285, + "rewards/rejected": -2.3691372871398926, + "step": 55 + }, + { + "epoch": 0.12562156503533106, + "grad_norm": 30.216317311158747, + "learning_rate": 4.990353313429303e-07, + "logits/chosen": -2.847761869430542, + "logits/rejected": -2.8407883644104004, + "logps/chosen": -1.0353600978851318, + "logps/rejected": -1.207129716873169, + "loss": 1.5827, + "rewards/accuracies": 0.5625, + "rewards/chosen": -2.0707201957702637, + "rewards/margins": 0.3435389995574951, + "rewards/rejected": -2.414259433746338, + "step": 60 + }, + { + "epoch": 0.1360900287882753, + "grad_norm": 25.74208520684934, + "learning_rate": 4.980652179769217e-07, + "logits/chosen": -2.8678090572357178, + "logits/rejected": -2.89147686958313, + "logps/chosen": -1.0218393802642822, + "logps/rejected": -1.196505069732666, + "loss": 1.5964, + "rewards/accuracies": 0.59375, + "rewards/chosen": -2.0436787605285645, + "rewards/margins": 0.3493315577507019, + "rewards/rejected": -2.393010139465332, + "step": 65 + }, + { + "epoch": 0.14655849254121958, + "grad_norm": 26.44073609906296, + "learning_rate": 4.967625656594781e-07, + "logits/chosen": -2.86854887008667, + "logits/rejected": -2.8683791160583496, + "logps/chosen": -1.1050409078598022, + "logps/rejected": -1.300787091255188, + "loss": 1.5446, + "rewards/accuracies": 0.637499988079071, + "rewards/chosen": -2.2100818157196045, + "rewards/margins": 0.391492635011673, + "rewards/rejected": -2.601574182510376, + "step": 70 + }, + { + "epoch": 0.15702695629416383, + "grad_norm": 32.621154621425745, + "learning_rate": 4.951291206355559e-07, + "logits/chosen": -2.816270351409912, + "logits/rejected": -2.7658441066741943, + "logps/chosen": -1.1289862394332886, + "logps/rejected": -1.42450749874115, + "loss": 1.5692, + "rewards/accuracies": 0.637499988079071, + "rewards/chosen": -2.257972478866577, + "rewards/margins": 0.5910428762435913, + "rewards/rejected": -2.8490149974823, + "step": 75 + }, + { + "epoch": 0.16749542004710807, + "grad_norm": 23.011756326022333, + "learning_rate": 4.93167072587771e-07, + "logits/chosen": -2.7463507652282715, + "logits/rejected": -2.741854667663574, + "logps/chosen": -1.118318796157837, + "logps/rejected": -1.486810564994812, + "loss": 1.5494, + "rewards/accuracies": 0.6875, + "rewards/chosen": -2.236637592315674, + "rewards/margins": 0.7369835376739502, + "rewards/rejected": -2.973621129989624, + "step": 80 + }, + { + "epoch": 0.17796388380005235, + "grad_norm": 28.536903966310312, + "learning_rate": 4.908790517010636e-07, + "logits/chosen": -2.8005614280700684, + "logits/rejected": -2.7525715827941895, + "logps/chosen": -1.1057350635528564, + "logps/rejected": -1.4478759765625, + "loss": 1.5328, + "rewards/accuracies": 0.637499988079071, + "rewards/chosen": -2.211470127105713, + "rewards/margins": 0.6842816472053528, + "rewards/rejected": -2.895751953125, + "step": 85 + }, + { + "epoch": 0.1884323475529966, + "grad_norm": 21.41681785684056, + "learning_rate": 4.882681251368548e-07, + "logits/chosen": -2.7789416313171387, + "logits/rejected": -2.7269153594970703, + "logps/chosen": -1.1642733812332153, + "logps/rejected": -1.5922167301177979, + "loss": 1.5065, + "rewards/accuracies": 0.6187499761581421, + "rewards/chosen": -2.3285467624664307, + "rewards/margins": 0.8558871150016785, + "rewards/rejected": -3.1844334602355957, + "step": 90 + }, + { + "epoch": 0.19890081130594087, + "grad_norm": 31.29059195928571, + "learning_rate": 4.853377929214243e-07, + "logits/chosen": -2.705059051513672, + "logits/rejected": -2.689885139465332, + "logps/chosen": -1.1885052919387817, + "logps/rejected": -1.503861665725708, + "loss": 1.5348, + "rewards/accuracies": 0.581250011920929, + "rewards/chosen": -2.3770105838775635, + "rewards/margins": 0.6307128667831421, + "rewards/rejected": -3.007723331451416, + "step": 95 + }, + { + "epoch": 0.2093692750588851, + "grad_norm": 29.067491467613685, + "learning_rate": 4.820919832540181e-07, + "logits/chosen": -2.776214122772217, + "logits/rejected": -2.7497711181640625, + "logps/chosen": -1.157089352607727, + "logps/rejected": -1.3819860219955444, + "loss": 1.5864, + "rewards/accuracies": 0.5562499761581421, + "rewards/chosen": -2.314178705215454, + "rewards/margins": 0.44979315996170044, + "rewards/rejected": -2.763972043991089, + "step": 100 + }, + { + "epoch": 0.21983773881182936, + "grad_norm": 23.067455098331433, + "learning_rate": 4.785350472409791e-07, + "logits/chosen": -2.7103278636932373, + "logits/rejected": -2.68127703666687, + "logps/chosen": -1.180944800376892, + "logps/rejected": -1.535935640335083, + "loss": 1.4346, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -2.361889600753784, + "rewards/margins": 0.7099816203117371, + "rewards/rejected": -3.071871280670166, + "step": 105 + }, + { + "epoch": 0.23030620256477363, + "grad_norm": 38.83165169666446, + "learning_rate": 4.7467175306295647e-07, + "logits/chosen": -2.6472716331481934, + "logits/rejected": -2.6239981651306152, + "logps/chosen": -1.1390531063079834, + "logps/rejected": -1.4835283756256104, + "loss": 1.4351, + "rewards/accuracies": 0.643750011920929, + "rewards/chosen": -2.278106212615967, + "rewards/margins": 0.6889505386352539, + "rewards/rejected": -2.9670567512512207, + "step": 110 + }, + { + "epoch": 0.24077466631771788, + "grad_norm": 28.07429755635963, + "learning_rate": 4.70507279583015e-07, + "logits/chosen": -2.6805942058563232, + "logits/rejected": -2.6706345081329346, + "logps/chosen": -1.1800041198730469, + "logps/rejected": -1.4017146825790405, + "loss": 1.527, + "rewards/accuracies": 0.668749988079071, + "rewards/chosen": -2.3600082397460938, + "rewards/margins": 0.4434211254119873, + "rewards/rejected": -2.803429365158081, + "step": 115 + }, + { + "epoch": 0.2512431300706621, + "grad_norm": 25.451878448448234, + "learning_rate": 4.6604720940421207e-07, + "logits/chosen": -2.695066213607788, + "logits/rejected": -2.6383955478668213, + "logps/chosen": -1.11952805519104, + "logps/rejected": -1.5505688190460205, + "loss": 1.5091, + "rewards/accuracies": 0.6187499761581421, + "rewards/chosen": -2.23905611038208, + "rewards/margins": 0.8620816469192505, + "rewards/rejected": -3.101137638092041, + "step": 120 + }, + { + "epoch": 0.26171159382360637, + "grad_norm": 28.525020675058567, + "learning_rate": 4.612975213859487e-07, + "logits/chosen": -2.5875375270843506, + "logits/rejected": -2.5485615730285645, + "logps/chosen": -1.1817591190338135, + "logps/rejected": -1.588836669921875, + "loss": 1.5135, + "rewards/accuracies": 0.6812499761581421, + "rewards/chosen": -2.363518238067627, + "rewards/margins": 0.8141552209854126, + "rewards/rejected": -3.17767333984375, + "step": 125 + }, + { + "epoch": 0.2721800575765506, + "grad_norm": 33.5709446027395, + "learning_rate": 4.5626458262912735e-07, + "logits/chosen": -2.6082234382629395, + "logits/rejected": -2.614490032196045, + "logps/chosen": -1.1771992444992065, + "logps/rejected": -1.5893863439559937, + "loss": 1.4422, + "rewards/accuracies": 0.7124999761581421, + "rewards/chosen": -2.354398488998413, + "rewards/margins": 0.8243740200996399, + "rewards/rejected": -3.1787726879119873, + "step": 130 + }, + { + "epoch": 0.2826485213294949, + "grad_norm": 26.6001483024502, + "learning_rate": 4.5095513994085974e-07, + "logits/chosen": -2.54782772064209, + "logits/rejected": -2.51760196685791, + "logps/chosen": -1.0749025344848633, + "logps/rejected": -1.5542266368865967, + "loss": 1.4927, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -2.1498050689697266, + "rewards/margins": 0.9586483240127563, + "rewards/rejected": -3.1084532737731934, + "step": 135 + }, + { + "epoch": 0.29311698508243916, + "grad_norm": 37.50133455368946, + "learning_rate": 4.453763107901675e-07, + "logits/chosen": -2.544795274734497, + "logits/rejected": -2.545607089996338, + "logps/chosen": -1.3422894477844238, + "logps/rejected": -1.5928326845169067, + "loss": 1.5489, + "rewards/accuracies": 0.612500011920929, + "rewards/chosen": -2.6845788955688477, + "rewards/margins": 0.5010861158370972, + "rewards/rejected": -3.1856653690338135, + "step": 140 + }, + { + "epoch": 0.3035854488353834, + "grad_norm": 30.749771656186557, + "learning_rate": 4.395355737667985e-07, + "logits/chosen": -2.5184309482574463, + "logits/rejected": -2.4989206790924072, + "logps/chosen": -1.229201078414917, + "logps/rejected": -1.7190883159637451, + "loss": 1.4308, + "rewards/accuracies": 0.6812499761581421, + "rewards/chosen": -2.458402156829834, + "rewards/margins": 0.9797741770744324, + "rewards/rejected": -3.4381766319274902, + "step": 145 + }, + { + "epoch": 0.31405391258832765, + "grad_norm": 34.705616703882306, + "learning_rate": 4.3344075855595097e-07, + "logits/chosen": -2.5644986629486084, + "logits/rejected": -2.5165212154388428, + "logps/chosen": -1.2188150882720947, + "logps/rejected": -1.596199631690979, + "loss": 1.4579, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -2.4376301765441895, + "rewards/margins": 0.7547694444656372, + "rewards/rejected": -3.192399263381958, + "step": 150 + }, + { + "epoch": 0.3245223763412719, + "grad_norm": 52.49121820937414, + "learning_rate": 4.271000354423425e-07, + "logits/chosen": -2.475571393966675, + "logits/rejected": -2.447605609893799, + "logps/chosen": -1.2845265865325928, + "logps/rejected": -1.8401352167129517, + "loss": 1.4002, + "rewards/accuracies": 0.6937500238418579, + "rewards/chosen": -2.5690531730651855, + "rewards/margins": 1.1112172603607178, + "rewards/rejected": -3.6802704334259033, + "step": 155 + }, + { + "epoch": 0.33499084009421615, + "grad_norm": 38.68489554173724, + "learning_rate": 4.2052190435769554e-07, + "logits/chosen": -2.445486545562744, + "logits/rejected": -2.4290173053741455, + "logps/chosen": -1.2640022039413452, + "logps/rejected": -1.90715754032135, + "loss": 1.4271, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -2.5280044078826904, + "rewards/margins": 1.2863105535507202, + "rewards/rejected": -3.8143150806427, + "step": 160 + }, + { + "epoch": 0.34545930384716045, + "grad_norm": 47.847635567656944, + "learning_rate": 4.137151834863213e-07, + "logits/chosen": -2.5186421871185303, + "logits/rejected": -2.476241111755371, + "logps/chosen": -1.2429786920547485, + "logps/rejected": -1.6412330865859985, + "loss": 1.3755, + "rewards/accuracies": 0.65625, + "rewards/chosen": -2.485957384109497, + "rewards/margins": 0.7965084314346313, + "rewards/rejected": -3.282466173171997, + "step": 165 + }, + { + "epoch": 0.3559277676001047, + "grad_norm": 39.55414229808935, + "learning_rate": 4.0668899744407567e-07, + "logits/chosen": -2.5068397521972656, + "logits/rejected": -2.458843231201172, + "logps/chosen": -1.2319917678833008, + "logps/rejected": -1.6991866827011108, + "loss": 1.3875, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -2.4639835357666016, + "rewards/margins": 0.9343900680541992, + "rewards/rejected": -3.3983733654022217, + "step": 170 + }, + { + "epoch": 0.36639623135304894, + "grad_norm": 45.87476259725973, + "learning_rate": 3.994527650465352e-07, + "logits/chosen": -2.4231743812561035, + "logits/rejected": -2.4120521545410156, + "logps/chosen": -1.2813334465026855, + "logps/rejected": -1.8030840158462524, + "loss": 1.3979, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -2.562666893005371, + "rewards/margins": 1.0435012578964233, + "rewards/rejected": -3.606168031692505, + "step": 175 + }, + { + "epoch": 0.3768646951059932, + "grad_norm": 52.555406414516895, + "learning_rate": 3.920161866827889e-07, + "logits/chosen": -2.438419818878174, + "logits/rejected": -2.4260263442993164, + "logps/chosen": -1.4097450971603394, + "logps/rejected": -1.9904677867889404, + "loss": 1.4005, + "rewards/accuracies": 0.7124999761581421, + "rewards/chosen": -2.8194901943206787, + "rewards/margins": 1.1614452600479126, + "rewards/rejected": -3.980935573577881, + "step": 180 + }, + { + "epoch": 0.38733315885893743, + "grad_norm": 45.270627593903136, + "learning_rate": 3.8438923131177237e-07, + "logits/chosen": -2.390608787536621, + "logits/rejected": -2.4067492485046387, + "logps/chosen": -1.4339230060577393, + "logps/rejected": -2.0207736492156982, + "loss": 1.3374, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -2.8678460121154785, + "rewards/margins": 1.173701524734497, + "rewards/rejected": -4.0415472984313965, + "step": 185 + }, + { + "epoch": 0.39780162261188173, + "grad_norm": 35.97699766429664, + "learning_rate": 3.765821230985757e-07, + "logits/chosen": -2.389159679412842, + "logits/rejected": -2.3969435691833496, + "logps/chosen": -1.4970250129699707, + "logps/rejected": -2.0378897190093994, + "loss": 1.3428, + "rewards/accuracies": 0.6937500238418579, + "rewards/chosen": -2.9940500259399414, + "rewards/margins": 1.081729531288147, + "rewards/rejected": -4.075779438018799, + "step": 190 + }, + { + "epoch": 0.408270086364826, + "grad_norm": 53.06912771848456, + "learning_rate": 3.6860532770864005e-07, + "logits/chosen": -2.3946540355682373, + "logits/rejected": -2.3885226249694824, + "logps/chosen": -1.5624958276748657, + "logps/rejected": -2.2428553104400635, + "loss": 1.4342, + "rewards/accuracies": 0.75, + "rewards/chosen": -3.1249916553497314, + "rewards/margins": 1.3607194423675537, + "rewards/rejected": -4.485710620880127, + "step": 195 + }, + { + "epoch": 0.4187385501177702, + "grad_norm": 46.84209895719228, + "learning_rate": 3.604695382782159e-07, + "logits/chosen": -2.2939867973327637, + "logits/rejected": -2.276329755783081, + "logps/chosen": -1.6615016460418701, + "logps/rejected": -2.154167652130127, + "loss": 1.3573, + "rewards/accuracies": 0.612500011920929, + "rewards/chosen": -3.3230032920837402, + "rewards/margins": 0.9853318333625793, + "rewards/rejected": -4.308335304260254, + "step": 200 + }, + { + "epoch": 0.42920701387071447, + "grad_norm": 63.174735320239826, + "learning_rate": 3.5218566107988867e-07, + "logits/chosen": -2.364863634109497, + "logits/rejected": -2.3571414947509766, + "logps/chosen": -1.7413444519042969, + "logps/rejected": -2.463257312774658, + "loss": 1.3484, + "rewards/accuracies": 0.65625, + "rewards/chosen": -3.4826889038085938, + "rewards/margins": 1.4438252449035645, + "rewards/rejected": -4.926514625549316, + "step": 205 + }, + { + "epoch": 0.4396754776236587, + "grad_norm": 53.13105715426384, + "learning_rate": 3.4376480090239047e-07, + "logits/chosen": -2.216089963912964, + "logits/rejected": -2.2170519828796387, + "logps/chosen": -1.8861106634140015, + "logps/rejected": -2.334871768951416, + "loss": 1.4309, + "rewards/accuracies": 0.6937500238418579, + "rewards/chosen": -3.772221326828003, + "rewards/margins": 0.8975218534469604, + "rewards/rejected": -4.669743537902832, + "step": 210 + }, + { + "epoch": 0.45014394137660296, + "grad_norm": 59.15829566376919, + "learning_rate": 3.3521824616429284e-07, + "logits/chosen": -2.225267171859741, + "logits/rejected": -2.250946521759033, + "logps/chosen": -1.865648627281189, + "logps/rejected": -2.464839220046997, + "loss": 1.3613, + "rewards/accuracies": 0.7562500238418579, + "rewards/chosen": -3.731297254562378, + "rewards/margins": 1.198380708694458, + "rewards/rejected": -4.929678440093994, + "step": 215 + }, + { + "epoch": 0.46061240512954726, + "grad_norm": 60.027916344260326, + "learning_rate": 3.265574537815398e-07, + "logits/chosen": -2.2151072025299072, + "logits/rejected": -2.2236618995666504, + "logps/chosen": -1.9221923351287842, + "logps/rejected": -2.5661563873291016, + "loss": 1.2543, + "rewards/accuracies": 0.731249988079071, + "rewards/chosen": -3.8443846702575684, + "rewards/margins": 1.2879277467727661, + "rewards/rejected": -5.132312774658203, + "step": 220 + }, + { + "epoch": 0.4710808688824915, + "grad_norm": 58.043157481996744, + "learning_rate": 3.1779403380910425e-07, + "logits/chosen": -2.1862709522247314, + "logits/rejected": -2.1562201976776123, + "logps/chosen": -2.2186365127563477, + "logps/rejected": -2.7678346633911133, + "loss": 1.3579, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -4.437273025512695, + "rewards/margins": 1.0983966588974, + "rewards/rejected": -5.535669326782227, + "step": 225 + }, + { + "epoch": 0.48154933263543576, + "grad_norm": 61.982027045427564, + "learning_rate": 3.0893973387735683e-07, + "logits/chosen": -2.1588408946990967, + "logits/rejected": -2.1733763217926025, + "logps/chosen": -2.2563939094543457, + "logps/rejected": -2.928832530975342, + "loss": 1.2783, + "rewards/accuracies": 0.6937500238418579, + "rewards/chosen": -4.512787818908691, + "rewards/margins": 1.3448779582977295, + "rewards/rejected": -5.857665061950684, + "step": 230 + }, + { + "epoch": 0.49201779638838, + "grad_norm": 53.47910914358791, + "learning_rate": 3.000064234440111e-07, + "logits/chosen": -2.066746234893799, + "logits/rejected": -2.0809385776519775, + "logps/chosen": -2.1830739974975586, + "logps/rejected": -2.7093000411987305, + "loss": 1.3083, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -4.366147994995117, + "rewards/margins": 1.0524519681930542, + "rewards/rejected": -5.418600082397461, + "step": 235 + }, + { + "epoch": 0.5024862601413242, + "grad_norm": 51.26667753004939, + "learning_rate": 2.910060778827554e-07, + "logits/chosen": -2.0583231449127197, + "logits/rejected": -2.0678017139434814, + "logps/chosen": -2.203321933746338, + "logps/rejected": -2.9744162559509277, + "loss": 1.3103, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -4.406643867492676, + "rewards/margins": 1.5421884059906006, + "rewards/rejected": -5.9488325119018555, + "step": 240 + }, + { + "epoch": 0.5129547238942685, + "grad_norm": 54.582002602834834, + "learning_rate": 2.8195076242990116e-07, + "logits/chosen": -2.076049327850342, + "logits/rejected": -2.105649948120117, + "logps/chosen": -2.2140307426452637, + "logps/rejected": -3.014204740524292, + "loss": 1.242, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -4.428061485290527, + "rewards/margins": 1.6003482341766357, + "rewards/rejected": -6.028409481048584, + "step": 245 + }, + { + "epoch": 0.5234231876472127, + "grad_norm": 65.50612310325059, + "learning_rate": 2.7285261601056697e-07, + "logits/chosen": -1.9772818088531494, + "logits/rejected": -2.0134644508361816, + "logps/chosen": -2.4714977741241455, + "logps/rejected": -3.3275609016418457, + "loss": 1.3888, + "rewards/accuracies": 0.737500011920929, + "rewards/chosen": -4.942995548248291, + "rewards/margins": 1.7121270895004272, + "rewards/rejected": -6.655121803283691, + "step": 250 + }, + { + "epoch": 0.533891651400157, + "grad_norm": 67.4992864768084, + "learning_rate": 2.6372383496608186e-07, + "logits/chosen": -2.059023141860962, + "logits/rejected": -2.0484743118286133, + "logps/chosen": -2.5257086753845215, + "logps/rejected": -3.2463812828063965, + "loss": 1.3315, + "rewards/accuracies": 0.6812499761581421, + "rewards/chosen": -5.051417350769043, + "rewards/margins": 1.4413448572158813, + "rewards/rejected": -6.492762565612793, + "step": 255 + }, + { + "epoch": 0.5443601151531012, + "grad_norm": 69.13744141717783, + "learning_rate": 2.5457665670441937e-07, + "logits/chosen": -2.006592273712158, + "logits/rejected": -2.014500141143799, + "logps/chosen": -2.560147762298584, + "logps/rejected": -3.3215479850769043, + "loss": 1.3082, + "rewards/accuracies": 0.71875, + "rewards/chosen": -5.120295524597168, + "rewards/margins": 1.5228006839752197, + "rewards/rejected": -6.643095970153809, + "step": 260 + }, + { + "epoch": 0.5548285789060455, + "grad_norm": 58.072442212510744, + "learning_rate": 2.454233432955807e-07, + "logits/chosen": -1.9831031560897827, + "logits/rejected": -1.9949731826782227, + "logps/chosen": -2.81583833694458, + "logps/rejected": -3.404637575149536, + "loss": 1.3035, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -5.63167667388916, + "rewards/margins": 1.1775987148284912, + "rewards/rejected": -6.809275150299072, + "step": 265 + }, + { + "epoch": 0.5652970426589898, + "grad_norm": 87.65400359085467, + "learning_rate": 2.3627616503391812e-07, + "logits/chosen": -2.0101704597473145, + "logits/rejected": -2.017418384552002, + "logps/chosen": -2.8499555587768555, + "logps/rejected": -3.4598793983459473, + "loss": 1.2368, + "rewards/accuracies": 0.71875, + "rewards/chosen": -5.699911117553711, + "rewards/margins": 1.219847559928894, + "rewards/rejected": -6.9197587966918945, + "step": 270 + }, + { + "epoch": 0.575765506411934, + "grad_norm": 45.097233149719436, + "learning_rate": 2.2714738398943308e-07, + "logits/chosen": -1.9900894165039062, + "logits/rejected": -1.990896224975586, + "logps/chosen": -2.661165714263916, + "logps/rejected": -3.678067445755005, + "loss": 1.1497, + "rewards/accuracies": 0.78125, + "rewards/chosen": -5.322331428527832, + "rewards/margins": 2.033804416656494, + "rewards/rejected": -7.35613489151001, + "step": 275 + }, + { + "epoch": 0.5862339701648783, + "grad_norm": 65.10418020833369, + "learning_rate": 2.1804923757009882e-07, + "logits/chosen": -2.0073654651641846, + "logits/rejected": -2.005659580230713, + "logps/chosen": -2.6216259002685547, + "logps/rejected": -3.4922943115234375, + "loss": 1.3131, + "rewards/accuracies": 0.71875, + "rewards/chosen": -5.243251800537109, + "rewards/margins": 1.7413362264633179, + "rewards/rejected": -6.984588623046875, + "step": 280 + }, + { + "epoch": 0.5967024339178225, + "grad_norm": 61.639250570571264, + "learning_rate": 2.089939221172446e-07, + "logits/chosen": -1.9917762279510498, + "logits/rejected": -1.982318639755249, + "logps/chosen": -2.5954482555389404, + "logps/rejected": -3.323718309402466, + "loss": 1.2697, + "rewards/accuracies": 0.737500011920929, + "rewards/chosen": -5.190896511077881, + "rewards/margins": 1.4565393924713135, + "rewards/rejected": -6.647436618804932, + "step": 285 + }, + { + "epoch": 0.6071708976707668, + "grad_norm": 69.64758924542811, + "learning_rate": 1.9999357655598891e-07, + "logits/chosen": -1.9363685846328735, + "logits/rejected": -1.9144127368927002, + "logps/chosen": -2.655473232269287, + "logps/rejected": -3.300741195678711, + "loss": 1.2942, + "rewards/accuracies": 0.71875, + "rewards/chosen": -5.310946464538574, + "rewards/margins": 1.2905361652374268, + "rewards/rejected": -6.601482391357422, + "step": 290 + }, + { + "epoch": 0.6176393614237111, + "grad_norm": 72.7122045465263, + "learning_rate": 1.9106026612264315e-07, + "logits/chosen": -1.9503307342529297, + "logits/rejected": -1.938823938369751, + "logps/chosen": -2.770573139190674, + "logps/rejected": -3.3855674266815186, + "loss": 1.2444, + "rewards/accuracies": 0.706250011920929, + "rewards/chosen": -5.541146278381348, + "rewards/margins": 1.2299891710281372, + "rewards/rejected": -6.771134853363037, + "step": 295 + }, + { + "epoch": 0.6281078251766553, + "grad_norm": 79.46704807847372, + "learning_rate": 1.8220596619089573e-07, + "logits/chosen": -1.8155113458633423, + "logits/rejected": -1.800158143043518, + "logps/chosen": -2.862525463104248, + "logps/rejected": -3.680985927581787, + "loss": 1.2954, + "rewards/accuracies": 0.737500011920929, + "rewards/chosen": -5.725050926208496, + "rewards/margins": 1.6369216442108154, + "rewards/rejected": -7.361971855163574, + "step": 300 + }, + { + "epoch": 0.6385762889295996, + "grad_norm": 77.33985147922212, + "learning_rate": 1.7344254621846017e-07, + "logits/chosen": -1.9133965969085693, + "logits/rejected": -1.9274393320083618, + "logps/chosen": -2.733630657196045, + "logps/rejected": -3.405717134475708, + "loss": 1.2303, + "rewards/accuracies": 0.731249988079071, + "rewards/chosen": -5.46726131439209, + "rewards/margins": 1.3441731929779053, + "rewards/rejected": -6.811434268951416, + "step": 305 + }, + { + "epoch": 0.6490447526825438, + "grad_norm": 87.53372002667572, + "learning_rate": 1.647817538357072e-07, + "logits/chosen": -1.8388316631317139, + "logits/rejected": -1.8692519664764404, + "logps/chosen": -2.9406466484069824, + "logps/rejected": -3.684401035308838, + "loss": 1.2409, + "rewards/accuracies": 0.737500011920929, + "rewards/chosen": -5.881293296813965, + "rewards/margins": 1.4875091314315796, + "rewards/rejected": -7.368802070617676, + "step": 310 + }, + { + "epoch": 0.6595132164354881, + "grad_norm": 63.982977249408776, + "learning_rate": 1.562351990976095e-07, + "logits/chosen": -1.9070667028427124, + "logits/rejected": -1.91787850856781, + "logps/chosen": -2.804577350616455, + "logps/rejected": -3.586026668548584, + "loss": 1.2734, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -5.60915470123291, + "rewards/margins": 1.5628979206085205, + "rewards/rejected": -7.172053337097168, + "step": 315 + }, + { + "epoch": 0.6699816801884323, + "grad_norm": 68.2071236663429, + "learning_rate": 1.478143389201113e-07, + "logits/chosen": -1.9229049682617188, + "logits/rejected": -1.9212894439697266, + "logps/chosen": -2.7598495483398438, + "logps/rejected": -3.6366069316864014, + "loss": 1.173, + "rewards/accuracies": 0.737500011920929, + "rewards/chosen": -5.5196990966796875, + "rewards/margins": 1.7535146474838257, + "rewards/rejected": -7.273213863372803, + "step": 320 + }, + { + "epoch": 0.6804501439413766, + "grad_norm": 82.89442348714914, + "learning_rate": 1.3953046172178413e-07, + "logits/chosen": -1.8585288524627686, + "logits/rejected": -1.8806421756744385, + "logps/chosen": -3.1847102642059326, + "logps/rejected": -4.028578758239746, + "loss": 1.319, + "rewards/accuracies": 0.6812499761581421, + "rewards/chosen": -6.369420528411865, + "rewards/margins": 1.6877357959747314, + "rewards/rejected": -8.057157516479492, + "step": 325 + }, + { + "epoch": 0.6909186076943209, + "grad_norm": 63.652198082327615, + "learning_rate": 1.3139467229135998e-07, + "logits/chosen": -1.8519096374511719, + "logits/rejected": -1.8569042682647705, + "logps/chosen": -3.092832565307617, + "logps/rejected": -4.055886268615723, + "loss": 1.2226, + "rewards/accuracies": 0.737500011920929, + "rewards/chosen": -6.185665130615234, + "rewards/margins": 1.9261070489883423, + "rewards/rejected": -8.111772537231445, + "step": 330 + }, + { + "epoch": 0.7013870714472651, + "grad_norm": 88.02707766997058, + "learning_rate": 1.2341787690142435e-07, + "logits/chosen": -1.8144184350967407, + "logits/rejected": -1.8429317474365234, + "logps/chosen": -3.2898623943328857, + "logps/rejected": -4.14624547958374, + "loss": 1.2802, + "rewards/accuracies": 0.706250011920929, + "rewards/chosen": -6.5797247886657715, + "rewards/margins": 1.7127659320831299, + "rewards/rejected": -8.29249095916748, + "step": 335 + }, + { + "epoch": 0.7118555352002094, + "grad_norm": 81.51547237456022, + "learning_rate": 1.1561076868822755e-07, + "logits/chosen": -1.8092453479766846, + "logits/rejected": -1.884722113609314, + "logps/chosen": -3.1887125968933105, + "logps/rejected": -4.263826370239258, + "loss": 1.2472, + "rewards/accuracies": 0.7562500238418579, + "rewards/chosen": -6.377425193786621, + "rewards/margins": 2.1502270698547363, + "rewards/rejected": -8.527652740478516, + "step": 340 + }, + { + "epoch": 0.7223239989531536, + "grad_norm": 61.57880406452509, + "learning_rate": 1.0798381331721107e-07, + "logits/chosen": -1.8666706085205078, + "logits/rejected": -1.8893043994903564, + "logps/chosen": -3.0370490550994873, + "logps/rejected": -3.9561619758605957, + "loss": 1.255, + "rewards/accuracies": 0.768750011920929, + "rewards/chosen": -6.074098110198975, + "rewards/margins": 1.838226079940796, + "rewards/rejected": -7.912323951721191, + "step": 345 + }, + { + "epoch": 0.7327924627060979, + "grad_norm": 58.397329958567745, + "learning_rate": 1.0054723495346482e-07, + "logits/chosen": -1.895630121231079, + "logits/rejected": -1.9232200384140015, + "logps/chosen": -2.7903852462768555, + "logps/rejected": -3.5561904907226562, + "loss": 1.1821, + "rewards/accuracies": 0.7124999761581421, + "rewards/chosen": -5.580770492553711, + "rewards/margins": 1.5316107273101807, + "rewards/rejected": -7.1123809814453125, + "step": 350 + }, + { + "epoch": 0.7432609264590422, + "grad_norm": 52.74691249746145, + "learning_rate": 9.331100255592436e-08, + "logits/chosen": -1.8785375356674194, + "logits/rejected": -1.9092729091644287, + "logps/chosen": -2.7013652324676514, + "logps/rejected": -3.447122573852539, + "loss": 1.3265, + "rewards/accuracies": 0.706250011920929, + "rewards/chosen": -5.402730464935303, + "rewards/margins": 1.4915144443511963, + "rewards/rejected": -6.894245147705078, + "step": 355 + }, + { + "epoch": 0.7537293902119864, + "grad_norm": 65.82463759506157, + "learning_rate": 8.628481651367875e-08, + "logits/chosen": -1.8855186700820923, + "logits/rejected": -1.9008718729019165, + "logps/chosen": -2.6092782020568848, + "logps/rejected": -3.4059150218963623, + "loss": 1.2498, + "rewards/accuracies": 0.762499988079071, + "rewards/chosen": -5.2185564041137695, + "rewards/margins": 1.5932741165161133, + "rewards/rejected": -6.811830043792725, + "step": 360 + }, + { + "epoch": 0.7641978539649307, + "grad_norm": 62.941225991768896, + "learning_rate": 7.947809564230445e-08, + "logits/chosen": -1.9058167934417725, + "logits/rejected": -1.9186041355133057, + "logps/chosen": -2.606133222579956, + "logps/rejected": -3.5266499519348145, + "loss": 1.2337, + "rewards/accuracies": 0.7124999761581421, + "rewards/chosen": -5.212266445159912, + "rewards/margins": 1.8410332202911377, + "rewards/rejected": -7.053299903869629, + "step": 365 + }, + { + "epoch": 0.7746663177178749, + "grad_norm": 80.15726866517387, + "learning_rate": 7.289996455765748e-08, + "logits/chosen": -1.8724944591522217, + "logits/rejected": -1.888501524925232, + "logps/chosen": -2.636202573776245, + "logps/rejected": -3.564650058746338, + "loss": 1.1151, + "rewards/accuracies": 0.7875000238418579, + "rewards/chosen": -5.27240514755249, + "rewards/margins": 1.8568944931030273, + "rewards/rejected": -7.129300117492676, + "step": 370 + }, + { + "epoch": 0.7851347814708192, + "grad_norm": 85.74581196541867, + "learning_rate": 6.655924144404906e-08, + "logits/chosen": -1.8323078155517578, + "logits/rejected": -1.8531125783920288, + "logps/chosen": -2.757204532623291, + "logps/rejected": -3.6793594360351562, + "loss": 1.2422, + "rewards/accuracies": 0.71875, + "rewards/chosen": -5.514409065246582, + "rewards/margins": 1.8443094491958618, + "rewards/rejected": -7.3587188720703125, + "step": 375 + }, + { + "epoch": 0.7956032452237635, + "grad_norm": 65.11322516172854, + "learning_rate": 6.046442623320145e-08, + "logits/chosen": -1.881985068321228, + "logits/rejected": -1.9223979711532593, + "logps/chosen": -2.7589480876922607, + "logps/rejected": -3.593773603439331, + "loss": 1.2412, + "rewards/accuracies": 0.731249988079071, + "rewards/chosen": -5.5178961753845215, + "rewards/margins": 1.6696510314941406, + "rewards/rejected": -7.187547206878662, + "step": 380 + }, + { + "epoch": 0.8060717089767077, + "grad_norm": 70.17153962504261, + "learning_rate": 5.4623689209832484e-08, + "logits/chosen": -1.8261827230453491, + "logits/rejected": -1.8423763513565063, + "logps/chosen": -2.850209951400757, + "logps/rejected": -3.670152187347412, + "loss": 1.2011, + "rewards/accuracies": 0.731249988079071, + "rewards/chosen": -5.700419902801514, + "rewards/margins": 1.639884352684021, + "rewards/rejected": -7.340304374694824, + "step": 385 + }, + { + "epoch": 0.816540172729652, + "grad_norm": 68.38146040720316, + "learning_rate": 4.904486005914027e-08, + "logits/chosen": -1.8114173412322998, + "logits/rejected": -1.8267343044281006, + "logps/chosen": -3.0265450477600098, + "logps/rejected": -3.899582624435425, + "loss": 1.187, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -6.0530900955200195, + "rewards/margins": 1.7460750341415405, + "rewards/rejected": -7.79916524887085, + "step": 390 + }, + { + "epoch": 0.8270086364825961, + "grad_norm": 77.09446527435051, + "learning_rate": 4.373541737087263e-08, + "logits/chosen": -1.7677643299102783, + "logits/rejected": -1.80731999874115, + "logps/chosen": -2.9306697845458984, + "logps/rejected": -3.8741061687469482, + "loss": 1.2481, + "rewards/accuracies": 0.75, + "rewards/chosen": -5.861339569091797, + "rewards/margins": 1.8868728876113892, + "rewards/rejected": -7.7482123374938965, + "step": 395 + }, + { + "epoch": 0.8374771002355405, + "grad_norm": 71.1284274145702, + "learning_rate": 3.8702478614051345e-08, + "logits/chosen": -1.7906978130340576, + "logits/rejected": -1.8041505813598633, + "logps/chosen": -2.976975202560425, + "logps/rejected": -3.7604854106903076, + "loss": 1.2816, + "rewards/accuracies": 0.75, + "rewards/chosen": -5.95395040512085, + "rewards/margins": 1.5670208930969238, + "rewards/rejected": -7.520970821380615, + "step": 400 + }, + { + "epoch": 0.8374771002355405, + "eval_logits/chosen": -1.8679399490356445, + "eval_logits/rejected": -1.879980444908142, + "eval_logps/chosen": -2.9611897468566895, + "eval_logps/rejected": -3.90179443359375, + "eval_loss": 1.2016023397445679, + "eval_rewards/accuracies": 0.7379999756813049, + "eval_rewards/chosen": -5.922379493713379, + "eval_rewards/margins": 1.8812097311019897, + "eval_rewards/rejected": -7.8035888671875, + "eval_runtime": 171.6266, + "eval_samples_per_second": 11.653, + "eval_steps_per_second": 0.728, + "step": 400 + }, + { + "epoch": 0.8479455639884846, + "grad_norm": 63.83271294085232, + "learning_rate": 3.3952790595787986e-08, + "logits/chosen": -1.8161646127700806, + "logits/rejected": -1.8377959728240967, + "logps/chosen": -3.039977788925171, + "logps/rejected": -3.8821442127227783, + "loss": 1.1734, + "rewards/accuracies": 0.71875, + "rewards/chosen": -6.079955577850342, + "rewards/margins": 1.6843324899673462, + "rewards/rejected": -7.764288425445557, + "step": 405 + }, + { + "epoch": 0.8584140277414289, + "grad_norm": 58.30518411516352, + "learning_rate": 2.9492720416985e-08, + "logits/chosen": -1.7875831127166748, + "logits/rejected": -1.8193409442901611, + "logps/chosen": -3.0099644660949707, + "logps/rejected": -3.809281587600708, + "loss": 1.2211, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -6.019928932189941, + "rewards/margins": 1.5986337661743164, + "rewards/rejected": -7.618563175201416, + "step": 410 + }, + { + "epoch": 0.8688824914943732, + "grad_norm": 75.30650643769822, + "learning_rate": 2.5328246937043525e-08, + "logits/chosen": -1.8199684619903564, + "logits/rejected": -1.8271119594573975, + "logps/chosen": -3.101931571960449, + "logps/rejected": -3.806241273880005, + "loss": 1.2569, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -6.203863143920898, + "rewards/margins": 1.4086189270019531, + "rewards/rejected": -7.61248254776001, + "step": 415 + }, + { + "epoch": 0.8793509552473174, + "grad_norm": 78.83242409965422, + "learning_rate": 2.1464952759020856e-08, + "logits/chosen": -1.7683451175689697, + "logits/rejected": -1.7631126642227173, + "logps/chosen": -2.9747061729431152, + "logps/rejected": -4.003817558288574, + "loss": 1.1673, + "rewards/accuracies": 0.7124999761581421, + "rewards/chosen": -5.9494123458862305, + "rewards/margins": 2.0582237243652344, + "rewards/rejected": -8.007635116577148, + "step": 420 + }, + { + "epoch": 0.8898194190002617, + "grad_norm": 78.91890630683983, + "learning_rate": 1.7908016745981856e-08, + "logits/chosen": -1.794054627418518, + "logits/rejected": -1.823462724685669, + "logps/chosen": -2.8728091716766357, + "logps/rejected": -3.8705430030822754, + "loss": 1.1418, + "rewards/accuracies": 0.7562500238418579, + "rewards/chosen": -5.7456183433532715, + "rewards/margins": 1.9954681396484375, + "rewards/rejected": -7.741086006164551, + "step": 425 + }, + { + "epoch": 0.9002878827532059, + "grad_norm": 77.02994695164918, + "learning_rate": 1.4662207078575684e-08, + "logits/chosen": -1.777092695236206, + "logits/rejected": -1.8050721883773804, + "logps/chosen": -3.0531764030456543, + "logps/rejected": -3.9285216331481934, + "loss": 1.1231, + "rewards/accuracies": 0.706250011920929, + "rewards/chosen": -6.106352806091309, + "rewards/margins": 1.7506898641586304, + "rewards/rejected": -7.857043266296387, + "step": 430 + }, + { + "epoch": 0.9107563465061502, + "grad_norm": 78.61899751414452, + "learning_rate": 1.1731874863145142e-08, + "logits/chosen": -1.783837914466858, + "logits/rejected": -1.7817462682724, + "logps/chosen": -2.908364772796631, + "logps/rejected": -3.625439405441284, + "loss": 1.1821, + "rewards/accuracies": 0.6625000238418579, + "rewards/chosen": -5.816729545593262, + "rewards/margins": 1.4341497421264648, + "rewards/rejected": -7.250878810882568, + "step": 435 + }, + { + "epoch": 0.9212248102590945, + "grad_norm": 94.12554697658324, + "learning_rate": 9.12094829893642e-09, + "logits/chosen": -1.835268259048462, + "logits/rejected": -1.8342323303222656, + "logps/chosen": -3.0066416263580322, + "logps/rejected": -3.7469406127929688, + "loss": 1.209, + "rewards/accuracies": 0.6937500238418579, + "rewards/chosen": -6.0132832527160645, + "rewards/margins": 1.4805978536605835, + "rewards/rejected": -7.4938812255859375, + "step": 440 + }, + { + "epoch": 0.9316932740120387, + "grad_norm": 78.08601822772991, + "learning_rate": 6.832927412229017e-09, + "logits/chosen": -1.7435308694839478, + "logits/rejected": -1.7644704580307007, + "logps/chosen": -3.0101089477539062, + "logps/rejected": -3.985473155975342, + "loss": 1.2211, + "rewards/accuracies": 0.7437499761581421, + "rewards/chosen": -6.0202178955078125, + "rewards/margins": 1.9507286548614502, + "rewards/rejected": -7.970946311950684, + "step": 445 + }, + { + "epoch": 0.942161737764983, + "grad_norm": 93.57542246833127, + "learning_rate": 4.8708793644441086e-09, + "logits/chosen": -1.7990074157714844, + "logits/rejected": -1.8333011865615845, + "logps/chosen": -2.7810821533203125, + "logps/rejected": -3.82600474357605, + "loss": 1.1351, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -5.562164306640625, + "rewards/margins": 2.08984375, + "rewards/rejected": -7.6520094871521, + "step": 450 + }, + { + "epoch": 0.9526302015179272, + "grad_norm": 70.87010486644522, + "learning_rate": 3.2374343405217884e-09, + "logits/chosen": -1.7927166223526, + "logits/rejected": -1.7781349420547485, + "logps/chosen": -2.9593749046325684, + "logps/rejected": -3.826608180999756, + "loss": 1.2607, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -5.918749809265137, + "rewards/margins": 1.7344669103622437, + "rewards/rejected": -7.653216361999512, + "step": 455 + }, + { + "epoch": 0.9630986652708715, + "grad_norm": 93.5003333433491, + "learning_rate": 1.9347820230782295e-09, + "logits/chosen": -1.7764625549316406, + "logits/rejected": -1.8235756158828735, + "logps/chosen": -2.923929214477539, + "logps/rejected": -3.9943060874938965, + "loss": 1.2149, + "rewards/accuracies": 0.7875000238418579, + "rewards/chosen": -5.847858428955078, + "rewards/margins": 2.1407532691955566, + "rewards/rejected": -7.988612174987793, + "step": 460 + }, + { + "epoch": 0.9735671290238157, + "grad_norm": 58.15978819844721, + "learning_rate": 9.64668657069706e-10, + "logits/chosen": -1.8141473531723022, + "logits/rejected": -1.805326223373413, + "logps/chosen": -2.911797046661377, + "logps/rejected": -4.0116353034973145, + "loss": 1.1457, + "rewards/accuracies": 0.7562500238418579, + "rewards/chosen": -5.823594093322754, + "rewards/margins": 2.1996755599975586, + "rewards/rejected": -8.023270606994629, + "step": 465 + }, + { + "epoch": 0.98403559277676, + "grad_norm": 66.91626514842031, + "learning_rate": 3.2839470889836627e-10, + "logits/chosen": -1.7597990036010742, + "logits/rejected": -1.7440340518951416, + "logps/chosen": -2.7474265098571777, + "logps/rejected": -3.8517417907714844, + "loss": 1.1939, + "rewards/accuracies": 0.7437499761581421, + "rewards/chosen": -5.4948530197143555, + "rewards/margins": 2.2086310386657715, + "rewards/rejected": -7.703483581542969, + "step": 470 + }, + { + "epoch": 0.9945040565297043, + "grad_norm": 95.9824717757533, + "learning_rate": 2.6813123097352287e-11, + "logits/chosen": -1.8236198425292969, + "logits/rejected": -1.8562953472137451, + "logps/chosen": -2.951925754547119, + "logps/rejected": -3.691092014312744, + "loss": 1.1781, + "rewards/accuracies": 0.6937500238418579, + "rewards/chosen": -5.903851509094238, + "rewards/margins": 1.4783318042755127, + "rewards/rejected": -7.382184028625488, + "step": 475 + }, + { + "epoch": 0.998691442030882, + "step": 477, + "total_flos": 0.0, + "train_loss": 1.3635074789918944, + "train_runtime": 22192.2901, + "train_samples_per_second": 2.755, + "train_steps_per_second": 0.021 + } + ], + "logging_steps": 5, + "max_steps": 477, + "num_input_tokens_seen": 0, + "num_train_epochs": 1, + "save_steps": 50000, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": true + }, + "attributes": {} + } + }, + "total_flos": 0.0, + "train_batch_size": 2, + "trial_name": null, + "trial_params": null +} diff --git a/training_args.bin b/training_args.bin new file mode 100644 index 0000000..951734f --- /dev/null +++ b/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a2a47a006c4220bd1e5885e96faa278bd2f425f24ac4cf37026623a4018d5f1a +size 6648