初始化项目，由ModelHub XC社区提供模型
Model: MusaKlair/pythia410m-dpo-beta0.1 Source: Original Platform
This commit is contained in:
ModelHub XC
35
.gitattributes
vendored
Normal file
35
.gitattributes
vendored
Normal file
@@ -0,0 +1,35 @@
|
|||||||
|
*.7z filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.arrow filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.bin filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.bz2 filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.ckpt filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.ftz filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.gz filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.h5 filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.joblib filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.lfs.* filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.mlmodel filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.model filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.msgpack filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.npy filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.npz filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.onnx filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.ot filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.parquet filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.pb filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.pickle filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.pkl filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.pt filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.pth filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.rar filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.safetensors filter=lfs diff=lfs merge=lfs -text
|
||||||
|
saved_model/**/* filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.tar.* filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.tar filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.tflite filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.tgz filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.wasm filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.xz filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.zip filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.zst filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
||||||
69
README.md
Normal file
69
README.md
Normal file
@@ -0,0 +1,69 @@
|
|||||||
|
---
|
||||||
|
base_model: EleutherAI/pythia-410m
|
||||||
|
library_name: transformers
|
||||||
|
model_name: pythia410m-dpo-beta0.1
|
||||||
|
tags:
|
||||||
|
- generated_from_trainer
|
||||||
|
- trl
|
||||||
|
- dpo
|
||||||
|
licence: license
|
||||||
|
---
|
||||||
|
|
||||||
|
# Model Card for pythia410m-dpo-beta0.1
|
||||||
|
|
||||||
|
This model is a fine-tuned version of [EleutherAI/pythia-410m](https://huggingface.co/EleutherAI/pythia-410m).
|
||||||
|
It has been trained using [TRL](https://github.com/huggingface/trl).
|
||||||
|
|
||||||
|
## Quick start
|
||||||
|
|
||||||
|
```python
|
||||||
|
from transformers import pipeline
|
||||||
|
|
||||||
|
question = "If you had a time machine, but could only go to the past or the future once and never return, which would you choose and why?"
|
||||||
|
generator = pipeline("text-generation", model="None", device="cuda")
|
||||||
|
output = generator([{"role": "user", "content": question}], max_new_tokens=128, return_full_text=False)[0]
|
||||||
|
print(output["generated_text"])
|
||||||
|
```
|
||||||
|
|
||||||
|
## Training procedure
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
This model was trained with DPO, a method introduced in [Direct Preference Optimization: Your Language Model is Secretly a Reward Model](https://huggingface.co/papers/2305.18290).
|
||||||
|
|
||||||
|
### Framework versions
|
||||||
|
|
||||||
|
- TRL: 1.7.1
|
||||||
|
- Transformers: 5.13.0
|
||||||
|
- Pytorch: 2.10.0+cu128
|
||||||
|
- Datasets: 5.0.0
|
||||||
|
- Tokenizers: 0.22.2
|
||||||
|
|
||||||
|
## Citations
|
||||||
|
|
||||||
|
Cite DPO as:
|
||||||
|
|
||||||
|
```bibtex
|
||||||
|
@inproceedings{rafailov2023direct,
|
||||||
|
title = {{Direct Preference Optimization: Your Language Model is Secretly a Reward Model}},
|
||||||
|
author = {Rafael Rafailov and Archit Sharma and Eric Mitchell and Christopher D. Manning and Stefano Ermon and Chelsea Finn},
|
||||||
|
year = 2023,
|
||||||
|
booktitle = {Advances in Neural Information Processing Systems 36: Annual Conference on Neural Information Processing Systems 2023, NeurIPS 2023, New Orleans, LA, USA, December 10 - 16, 2023},
|
||||||
|
url = {http://papers.nips.cc/paper_files/paper/2023/hash/a85b405ed65c6477a4fe8302b5e06ce7-Abstract-Conference.html},
|
||||||
|
editor = {Alice Oh and Tristan Naumann and Amir Globerson and Kate Saenko and Moritz Hardt and Sergey Levine},
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
Cite TRL as:
|
||||||
|
|
||||||
|
```bibtex
|
||||||
|
@software{vonwerra2020trl,
|
||||||
|
title = {{TRL: Transformers Reinforcement Learning}},
|
||||||
|
author = {von Werra, Leandro and Belkada, Younes and Tunstall, Lewis and Beeching, Edward and Thrush, Tristan and Lambert, Nathan and Huang, Shengyi and Rasul, Kashif and Gallouédec, Quentin},
|
||||||
|
license = {Apache-2.0},
|
||||||
|
url = {https://github.com/huggingface/trl},
|
||||||
|
year = {2020}
|
||||||
|
}
|
||||||
|
```
|
||||||
33
checkpoint-100/config.json
Normal file
33
checkpoint-100/config.json
Normal file
@@ -0,0 +1,33 @@
|
|||||||
|
{
|
||||||
|
"architectures": [
|
||||||
|
"GPTNeoXForCausalLM"
|
||||||
|
],
|
||||||
|
"attention_bias": true,
|
||||||
|
"attention_dropout": 0.0,
|
||||||
|
"bos_token_id": 0,
|
||||||
|
"classifier_dropout": 0.1,
|
||||||
|
"dtype": "float32",
|
||||||
|
"eos_token_id": 0,
|
||||||
|
"hidden_act": "gelu",
|
||||||
|
"hidden_dropout": 0.0,
|
||||||
|
"hidden_size": 1024,
|
||||||
|
"initializer_range": 0.02,
|
||||||
|
"intermediate_size": 4096,
|
||||||
|
"is_decoder": false,
|
||||||
|
"layer_norm_eps": 1e-05,
|
||||||
|
"max_position_embeddings": 2048,
|
||||||
|
"model_type": "gpt_neox",
|
||||||
|
"num_attention_heads": 16,
|
||||||
|
"num_hidden_layers": 24,
|
||||||
|
"pad_token_id": 1,
|
||||||
|
"rope_parameters": {
|
||||||
|
"partial_rotary_factor": 0.25,
|
||||||
|
"rope_theta": 10000,
|
||||||
|
"rope_type": "default"
|
||||||
|
},
|
||||||
|
"tie_word_embeddings": false,
|
||||||
|
"transformers_version": "5.13.0",
|
||||||
|
"use_cache": false,
|
||||||
|
"use_parallel_residual": true,
|
||||||
|
"vocab_size": 50304
|
||||||
|
}
|
||||||
10
checkpoint-100/generation_config.json
Normal file
10
checkpoint-100/generation_config.json
Normal file
@@ -0,0 +1,10 @@
|
|||||||
|
{
|
||||||
|
"_from_model_config": true,
|
||||||
|
"bos_token_id": 0,
|
||||||
|
"eos_token_id": [
|
||||||
|
0
|
||||||
|
],
|
||||||
|
"pad_token_id": 1,
|
||||||
|
"transformers_version": "5.13.0",
|
||||||
|
"use_cache": true
|
||||||
|
}
|
||||||
3
checkpoint-100/model.safetensors
Normal file
3
checkpoint-100/model.safetensors
Normal file
@@ -0,0 +1,3 @@
|
|||||||
|
version https://git-lfs.github.com/spec/v1
|
||||||
|
oid sha256:52523e4ba80176489b3cf3caca86a4235ad0f23407d94ba80c9e2254b283f3e7
|
||||||
|
size 1621370224
|
||||||
3
checkpoint-100/optimizer.pt
Normal file
3
checkpoint-100/optimizer.pt
Normal file
@@ -0,0 +1,3 @@
|
|||||||
|
version https://git-lfs.github.com/spec/v1
|
||||||
|
oid sha256:bc50801e980db61cadded1f35221d179cf5ea7edb639ba03524a138b1656c0d1
|
||||||
|
size 3242916491
|
||||||
3
checkpoint-100/rng_state.pth
Normal file
3
checkpoint-100/rng_state.pth
Normal file
@@ -0,0 +1,3 @@
|
|||||||
|
version https://git-lfs.github.com/spec/v1
|
||||||
|
oid sha256:718a0f3db00824213036a2c0441849791319b7d9cf189065873bb26a7020738e
|
||||||
|
size 14645
|
||||||
3
checkpoint-100/scaler.pt
Normal file
3
checkpoint-100/scaler.pt
Normal file
@@ -0,0 +1,3 @@
|
|||||||
|
version https://git-lfs.github.com/spec/v1
|
||||||
|
oid sha256:c4393a84a3109995aa1202073b039b12062e3189ed89aa0b94ef0510ba843009
|
||||||
|
size 1383
|
||||||
3
checkpoint-100/scheduler.pt
Normal file
3
checkpoint-100/scheduler.pt
Normal file
@@ -0,0 +1,3 @@
|
|||||||
|
version https://git-lfs.github.com/spec/v1
|
||||||
|
oid sha256:da6acab1199ce8b95be17924d67d35a7688c7036095034a44b256466f55879dc
|
||||||
|
size 1465
|
||||||
250557
checkpoint-100/tokenizer.json
Normal file
250557
checkpoint-100/tokenizer.json
Normal file
File diff suppressed because it is too large
Load Diff
14
checkpoint-100/tokenizer_config.json
Normal file
14
checkpoint-100/tokenizer_config.json
Normal file
@@ -0,0 +1,14 @@
|
|||||||
|
{
|
||||||
|
"add_prefix_space": false,
|
||||||
|
"backend": "tokenizers",
|
||||||
|
"bos_token": "<|endoftext|>",
|
||||||
|
"eos_token": "<|endoftext|>",
|
||||||
|
"errors": "replace",
|
||||||
|
"is_local": false,
|
||||||
|
"local_files_only": false,
|
||||||
|
"model_max_length": 1000000000000000019884624838656,
|
||||||
|
"pad_token": "<|padding|>",
|
||||||
|
"tokenizer_class": "GPTNeoXTokenizer",
|
||||||
|
"trim_offsets": true,
|
||||||
|
"unk_token": "<|endoftext|>"
|
||||||
|
}
|
||||||
451
checkpoint-100/trainer_state.json
Normal file
451
checkpoint-100/trainer_state.json
Normal file
@@ -0,0 +1,451 @@
|
|||||||
|
{
|
||||||
|
"best_global_step": null,
|
||||||
|
"best_metric": null,
|
||||||
|
"best_model_checkpoint": null,
|
||||||
|
"epoch": 1.592,
|
||||||
|
"eval_steps": 50,
|
||||||
|
"global_step": 100,
|
||||||
|
"is_hyper_param_search": false,
|
||||||
|
"is_local_process_zero": true,
|
||||||
|
"is_world_process_zero": true,
|
||||||
|
"log_history": [
|
||||||
|
{
|
||||||
|
"entropy": 2.555654156208038,
|
||||||
|
"epoch": 0.08,
|
||||||
|
"grad_norm": 35.3809700012207,
|
||||||
|
"learning_rate": 2.8571428571428573e-06,
|
||||||
|
"logits/chosen": 0.4110352631849171,
|
||||||
|
"logits/rejected": 0.4300117739250047,
|
||||||
|
"logps/chosen": -157.81439809799195,
|
||||||
|
"logps/rejected": -150.4315113067627,
|
||||||
|
"loss": 0.6903757572174072,
|
||||||
|
"mean_token_accuracy": 0.45491820499300956,
|
||||||
|
"num_tokens": 63280.0,
|
||||||
|
"rewards/accuracies": 0.54375,
|
||||||
|
"rewards/chosen": 0.018655257727368736,
|
||||||
|
"rewards/margins": 0.006082923361100256,
|
||||||
|
"rewards/rejected": 0.012572333973366767,
|
||||||
|
"step": 5
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"entropy": 2.5844572484493256,
|
||||||
|
"epoch": 0.16,
|
||||||
|
"grad_norm": 47.283817291259766,
|
||||||
|
"learning_rate": 4.984280524733107e-06,
|
||||||
|
"logits/chosen": 0.2089756903127898,
|
||||||
|
"logits/rejected": 0.3923359114864852,
|
||||||
|
"logps/chosen": -160.2436939239502,
|
||||||
|
"logps/rejected": -154.7825647354126,
|
||||||
|
"loss": 0.7087126731872558,
|
||||||
|
"mean_token_accuracy": 0.4676915630698204,
|
||||||
|
"num_tokens": 128751.0,
|
||||||
|
"rewards/accuracies": 0.46875,
|
||||||
|
"rewards/chosen": 0.026459120609797538,
|
||||||
|
"rewards/margins": -0.018515753094106913,
|
||||||
|
"rewards/rejected": 0.04497487433254719,
|
||||||
|
"step": 10
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"entropy": 2.4940366342663767,
|
||||||
|
"epoch": 0.24,
|
||||||
|
"grad_norm": 48.20133972167969,
|
||||||
|
"learning_rate": 4.809698831278217e-06,
|
||||||
|
"logits/chosen": 0.12422276096059184,
|
||||||
|
"logits/rejected": 0.18387425940175633,
|
||||||
|
"logps/chosen": -151.31794481277467,
|
||||||
|
"logps/rejected": -163.5261468887329,
|
||||||
|
"loss": 0.6959157466888428,
|
||||||
|
"mean_token_accuracy": 0.47235521450638773,
|
||||||
|
"num_tokens": 194142.0,
|
||||||
|
"rewards/accuracies": 0.46875,
|
||||||
|
"rewards/chosen": -0.059232240472920236,
|
||||||
|
"rewards/margins": 0.021671066922135652,
|
||||||
|
"rewards/rejected": -0.0809033086989075,
|
||||||
|
"step": 15
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"entropy": 2.5471011281013487,
|
||||||
|
"epoch": 0.32,
|
||||||
|
"grad_norm": 42.294281005859375,
|
||||||
|
"learning_rate": 4.454578706170075e-06,
|
||||||
|
"logits/chosen": -0.030133471365837695,
|
||||||
|
"logits/rejected": 0.08451241878365641,
|
||||||
|
"logps/chosen": -135.41445970535278,
|
||||||
|
"logps/rejected": -135.83921966552734,
|
||||||
|
"loss": 0.6954083442687988,
|
||||||
|
"mean_token_accuracy": 0.4549576163291931,
|
||||||
|
"num_tokens": 254871.0,
|
||||||
|
"rewards/accuracies": 0.5375,
|
||||||
|
"rewards/chosen": -0.18437330080196263,
|
||||||
|
"rewards/margins": 0.07001338726840914,
|
||||||
|
"rewards/rejected": -0.2543866881169379,
|
||||||
|
"step": 20
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"entropy": 2.4626115679740908,
|
||||||
|
"epoch": 0.4,
|
||||||
|
"grad_norm": 53.21565628051758,
|
||||||
|
"learning_rate": 3.946678240449515e-06,
|
||||||
|
"logits/chosen": 0.035210923156354317,
|
||||||
|
"logits/rejected": 0.03131556176291432,
|
||||||
|
"logps/chosen": -147.02334337234498,
|
||||||
|
"logps/rejected": -139.0251603126526,
|
||||||
|
"loss": 0.7144542694091797,
|
||||||
|
"mean_token_accuracy": 0.4505583204329014,
|
||||||
|
"num_tokens": 317752.0,
|
||||||
|
"rewards/accuracies": 0.55,
|
||||||
|
"rewards/chosen": -0.3056703276000917,
|
||||||
|
"rewards/margins": 0.030420188792049885,
|
||||||
|
"rewards/rejected": -0.33609051471576096,
|
||||||
|
"step": 25
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"entropy": 2.418185669183731,
|
||||||
|
"epoch": 0.48,
|
||||||
|
"grad_norm": 50.63364791870117,
|
||||||
|
"learning_rate": 3.3256976548879183e-06,
|
||||||
|
"logits/chosen": -0.0063577626599184795,
|
||||||
|
"logits/rejected": 0.0660789555792278,
|
||||||
|
"logps/chosen": -151.3002492904663,
|
||||||
|
"logps/rejected": -150.41238498687744,
|
||||||
|
"loss": 0.6920054435729981,
|
||||||
|
"mean_token_accuracy": 0.445500810444355,
|
||||||
|
"num_tokens": 387452.0,
|
||||||
|
"rewards/accuracies": 0.55625,
|
||||||
|
"rewards/chosen": -0.2962045251857489,
|
||||||
|
"rewards/margins": 0.08783619347959756,
|
||||||
|
"rewards/rejected": -0.38404072066769,
|
||||||
|
"step": 30
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"entropy": 2.42596016228199,
|
||||||
|
"epoch": 0.56,
|
||||||
|
"grad_norm": 47.31837844848633,
|
||||||
|
"learning_rate": 2.6401761180929798e-06,
|
||||||
|
"logits/chosen": -0.04082900087497934,
|
||||||
|
"logits/rejected": 0.015806176234233794,
|
||||||
|
"logps/chosen": -182.25081090927125,
|
||||||
|
"logps/rejected": -175.18973140716554,
|
||||||
|
"loss": 0.6894076824188232,
|
||||||
|
"mean_token_accuracy": 0.46841963976621626,
|
||||||
|
"num_tokens": 454873.0,
|
||||||
|
"rewards/accuracies": 0.55625,
|
||||||
|
"rewards/chosen": -0.30482858400791885,
|
||||||
|
"rewards/margins": 0.07829219745472074,
|
||||||
|
"rewards/rejected": -0.38312078421004114,
|
||||||
|
"step": 35
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"entropy": 2.421294018626213,
|
||||||
|
"epoch": 0.64,
|
||||||
|
"grad_norm": 47.72428894042969,
|
||||||
|
"learning_rate": 1.9436976651092143e-06,
|
||||||
|
"logits/chosen": 0.04491766159482387,
|
||||||
|
"logits/rejected": 0.15141158319763592,
|
||||||
|
"logps/chosen": -160.55729579925537,
|
||||||
|
"logps/rejected": -161.91398558616638,
|
||||||
|
"loss": 0.7262070178985596,
|
||||||
|
"mean_token_accuracy": 0.4616738587617874,
|
||||||
|
"num_tokens": 518527.0,
|
||||||
|
"rewards/accuracies": 0.51875,
|
||||||
|
"rewards/chosen": -0.2627937063574791,
|
||||||
|
"rewards/margins": 0.037597383465617897,
|
||||||
|
"rewards/rejected": -0.30039109364151956,
|
||||||
|
"step": 40
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"entropy": 2.3493838876485826,
|
||||||
|
"epoch": 0.72,
|
||||||
|
"grad_norm": 41.70436096191406,
|
||||||
|
"learning_rate": 1.2907027822369006e-06,
|
||||||
|
"logits/chosen": 0.06111884454416239,
|
||||||
|
"logits/rejected": 0.1077471313778626,
|
||||||
|
"logps/chosen": -155.73900985717773,
|
||||||
|
"logps/rejected": -162.23499221801757,
|
||||||
|
"loss": 0.6567366123199463,
|
||||||
|
"mean_token_accuracy": 0.47065152823925016,
|
||||||
|
"num_tokens": 586496.0,
|
||||||
|
"rewards/accuracies": 0.5875,
|
||||||
|
"rewards/chosen": -0.15574295134283603,
|
||||||
|
"rewards/margins": 0.16263325293548406,
|
||||||
|
"rewards/rejected": -0.31837619938887657,
|
||||||
|
"step": 45
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"entropy": 2.3660752564668655,
|
||||||
|
"epoch": 0.8,
|
||||||
|
"grad_norm": 46.26735305786133,
|
||||||
|
"learning_rate": 7.322330470336314e-07,
|
||||||
|
"logits/chosen": 0.04797605992282007,
|
||||||
|
"logits/rejected": 0.137719508348569,
|
||||||
|
"logps/chosen": -165.56337423324584,
|
||||||
|
"logps/rejected": -165.80605907440184,
|
||||||
|
"loss": 0.6880872249603271,
|
||||||
|
"mean_token_accuracy": 0.4618512228131294,
|
||||||
|
"num_tokens": 659345.0,
|
||||||
|
"rewards/accuracies": 0.51875,
|
||||||
|
"rewards/chosen": -0.13928775684908032,
|
||||||
|
"rewards/margins": 0.09893286619335413,
|
||||||
|
"rewards/rejected": -0.238220629747957,
|
||||||
|
"step": 50
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.8,
|
||||||
|
"eval_entropy": 2.4305524230003357,
|
||||||
|
"eval_logits/chosen": 0.060791561005096514,
|
||||||
|
"eval_logits/rejected": 0.1157722840661163,
|
||||||
|
"eval_logps/chosen": -163.22620516967774,
|
||||||
|
"eval_logps/rejected": -155.69468914794922,
|
||||||
|
"eval_loss": 0.699152946472168,
|
||||||
|
"eval_mean_token_accuracy": 0.4591814261674881,
|
||||||
|
"eval_num_tokens": 659345.0,
|
||||||
|
"eval_rewards/accuracies": 0.523,
|
||||||
|
"eval_rewards/chosen": -0.17783968531899155,
|
||||||
|
"eval_rewards/margins": 0.07608089716732502,
|
||||||
|
"eval_rewards/rejected": -0.2539205798432231,
|
||||||
|
"eval_runtime": 204.6975,
|
||||||
|
"eval_samples_per_second": 4.885,
|
||||||
|
"eval_steps_per_second": 1.221,
|
||||||
|
"step": 50
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"entropy": 2.3940737187862395,
|
||||||
|
"epoch": 0.88,
|
||||||
|
"grad_norm": 48.0695915222168,
|
||||||
|
"learning_rate": 3.119414452281158e-07,
|
||||||
|
"logits/chosen": 0.07112344903443206,
|
||||||
|
"logits/rejected": 0.19535127089682655,
|
||||||
|
"logps/chosen": -150.24068050384523,
|
||||||
|
"logps/rejected": -145.48797540664674,
|
||||||
|
"loss": 0.697087287902832,
|
||||||
|
"mean_token_accuracy": 0.4701634831726551,
|
||||||
|
"num_tokens": 731475.0,
|
||||||
|
"rewards/accuracies": 0.56875,
|
||||||
|
"rewards/chosen": -0.14963094238191843,
|
||||||
|
"rewards/margins": 0.08578414842486382,
|
||||||
|
"rewards/rejected": -0.2354150922037661,
|
||||||
|
"step": 55
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"entropy": 2.357463392615318,
|
||||||
|
"epoch": 0.96,
|
||||||
|
"grad_norm": 48.10399627685547,
|
||||||
|
"learning_rate": 6.268021954544095e-08,
|
||||||
|
"logits/chosen": 0.01995007363434443,
|
||||||
|
"logits/rejected": 0.16687794622622143,
|
||||||
|
"logps/chosen": -158.93942432403566,
|
||||||
|
"logps/rejected": -166.66946430206298,
|
||||||
|
"loss": 0.6701779842376709,
|
||||||
|
"mean_token_accuracy": 0.4689980365335941,
|
||||||
|
"num_tokens": 797757.0,
|
||||||
|
"rewards/accuracies": 0.5375,
|
||||||
|
"rewards/chosen": -0.1631853505037725,
|
||||||
|
"rewards/margins": 0.13551276298239828,
|
||||||
|
"rewards/rejected": -0.2986981191672385,
|
||||||
|
"step": 60
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 1.0,
|
||||||
|
"eval_entropy": 2.431350471973419,
|
||||||
|
"eval_logits/chosen": 0.058027013522972216,
|
||||||
|
"eval_logits/rejected": 0.11281541237759458,
|
||||||
|
"eval_logps/chosen": -163.3362089691162,
|
||||||
|
"eval_logps/rejected": -155.8263526611328,
|
||||||
|
"eval_loss": 0.6985306143760681,
|
||||||
|
"eval_mean_token_accuracy": 0.45935997557640074,
|
||||||
|
"eval_num_tokens": 829784.0,
|
||||||
|
"eval_rewards/accuracies": 0.529,
|
||||||
|
"eval_rewards/chosen": -0.188840083822608,
|
||||||
|
"eval_rewards/margins": 0.07824680726230145,
|
||||||
|
"eval_rewards/rejected": -0.26708688925206664,
|
||||||
|
"eval_runtime": 204.726,
|
||||||
|
"eval_samples_per_second": 4.885,
|
||||||
|
"eval_steps_per_second": 1.221,
|
||||||
|
"step": 63
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"entropy": 2.402912601828575,
|
||||||
|
"epoch": 1.032,
|
||||||
|
"grad_norm": 32.69980239868164,
|
||||||
|
"learning_rate": 4.184239109116393e-06,
|
||||||
|
"logits/chosen": 0.05448504114962113,
|
||||||
|
"logits/rejected": 0.09865282891809334,
|
||||||
|
"logps/chosen": -146.60927033424377,
|
||||||
|
"logps/rejected": -172.2928113937378,
|
||||||
|
"loss": 0.3366573750972748,
|
||||||
|
"mean_token_accuracy": 0.4740424118936062,
|
||||||
|
"num_tokens": 27180.0,
|
||||||
|
"rewards/accuracies": 0.859375,
|
||||||
|
"rewards/chosen": 0.5033185752108693,
|
||||||
|
"rewards/margins": 1.529684765264392,
|
||||||
|
"rewards/rejected": -1.0263662077486515,
|
||||||
|
"step": 65
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"entropy": 2.3992972910404204,
|
||||||
|
"epoch": 1.112,
|
||||||
|
"grad_norm": 17.76884651184082,
|
||||||
|
"learning_rate": 4.006586590948141e-06,
|
||||||
|
"logits/chosen": 0.0026862717521520412,
|
||||||
|
"logits/rejected": 0.13353262495170462,
|
||||||
|
"logps/chosen": -153.0885206222534,
|
||||||
|
"logps/rejected": -156.96604356765747,
|
||||||
|
"loss": 0.3243739604949951,
|
||||||
|
"mean_token_accuracy": 0.47915840819478034,
|
||||||
|
"num_tokens": 86611.0,
|
||||||
|
"rewards/accuracies": 0.8625,
|
||||||
|
"rewards/chosen": 0.4796073419041932,
|
||||||
|
"rewards/margins": 1.6072665184736252,
|
||||||
|
"rewards/rejected": -1.1276591904461384,
|
||||||
|
"step": 70
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"entropy": 2.3780136823654177,
|
||||||
|
"epoch": 1.192,
|
||||||
|
"grad_norm": 22.48740005493164,
|
||||||
|
"learning_rate": 3.81608040719339e-06,
|
||||||
|
"logits/chosen": -0.11707000613882948,
|
||||||
|
"logits/rejected": -0.06640926333257077,
|
||||||
|
"logps/chosen": -153.8634229660034,
|
||||||
|
"logps/rejected": -163.86640148162843,
|
||||||
|
"loss": 0.35583436489105225,
|
||||||
|
"mean_token_accuracy": 0.4692669503390789,
|
||||||
|
"num_tokens": 153056.0,
|
||||||
|
"rewards/accuracies": 0.83125,
|
||||||
|
"rewards/chosen": 0.23908968223258853,
|
||||||
|
"rewards/margins": 1.5137944724410772,
|
||||||
|
"rewards/rejected": -1.2747048027813435,
|
||||||
|
"step": 75
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"entropy": 2.310171273350716,
|
||||||
|
"epoch": 1.272,
|
||||||
|
"grad_norm": 19.922534942626953,
|
||||||
|
"learning_rate": 3.6143458894413463e-06,
|
||||||
|
"logits/chosen": -0.3582654699758253,
|
||||||
|
"logits/rejected": -0.2914110555931979,
|
||||||
|
"logps/chosen": -131.75142288208008,
|
||||||
|
"logps/rejected": -160.25545768737794,
|
||||||
|
"loss": 0.33200573921203613,
|
||||||
|
"mean_token_accuracy": 0.4669229932129383,
|
||||||
|
"num_tokens": 215920.0,
|
||||||
|
"rewards/accuracies": 0.8875,
|
||||||
|
"rewards/chosen": 0.15461167003959417,
|
||||||
|
"rewards/margins": 1.571430729702115,
|
||||||
|
"rewards/rejected": -1.4168190509080887,
|
||||||
|
"step": 80
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"entropy": 2.211333890259266,
|
||||||
|
"epoch": 1.3519999999999999,
|
||||||
|
"grad_norm": 26.273008346557617,
|
||||||
|
"learning_rate": 3.403104165467883e-06,
|
||||||
|
"logits/chosen": -0.4359116041083054,
|
||||||
|
"logits/rejected": -0.3527264562702821,
|
||||||
|
"logps/chosen": -140.573939037323,
|
||||||
|
"logps/rejected": -165.2755883216858,
|
||||||
|
"loss": 0.3167553186416626,
|
||||||
|
"mean_token_accuracy": 0.49101626574993135,
|
||||||
|
"num_tokens": 287127.0,
|
||||||
|
"rewards/accuracies": 0.86875,
|
||||||
|
"rewards/chosen": 0.22212664941325783,
|
||||||
|
"rewards/margins": 1.8355195850133896,
|
||||||
|
"rewards/rejected": -1.6133929178118707,
|
||||||
|
"step": 85
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"entropy": 2.284930866956711,
|
||||||
|
"epoch": 1.432,
|
||||||
|
"grad_norm": 23.699182510375977,
|
||||||
|
"learning_rate": 3.184157475180208e-06,
|
||||||
|
"logits/chosen": -0.49555035845434314,
|
||||||
|
"logits/rejected": -0.4411274731208847,
|
||||||
|
"logps/chosen": -159.97724952697754,
|
||||||
|
"logps/rejected": -158.18697261810303,
|
||||||
|
"loss": 0.31732380390167236,
|
||||||
|
"mean_token_accuracy": 0.46273171752691267,
|
||||||
|
"num_tokens": 354068.0,
|
||||||
|
"rewards/accuracies": 0.9,
|
||||||
|
"rewards/chosen": 0.2169143119826913,
|
||||||
|
"rewards/margins": 1.6415132291615009,
|
||||||
|
"rewards/rejected": -1.4245989315211773,
|
||||||
|
"step": 90
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"entropy": 2.185201385617256,
|
||||||
|
"epoch": 1.512,
|
||||||
|
"grad_norm": 19.721302032470703,
|
||||||
|
"learning_rate": 2.9593737945414264e-06,
|
||||||
|
"logits/chosen": -0.6350667553105876,
|
||||||
|
"logits/rejected": -0.6234260760266512,
|
||||||
|
"logps/chosen": -140.15507984161377,
|
||||||
|
"logps/rejected": -165.66675672531127,
|
||||||
|
"loss": 0.3336572885513306,
|
||||||
|
"mean_token_accuracy": 0.46677022203803065,
|
||||||
|
"num_tokens": 421426.0,
|
||||||
|
"rewards/accuracies": 0.8375,
|
||||||
|
"rewards/chosen": 0.1235144454985857,
|
||||||
|
"rewards/margins": 1.6951029762625693,
|
||||||
|
"rewards/rejected": -1.5715885123237967,
|
||||||
|
"step": 95
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"entropy": 2.116330775618553,
|
||||||
|
"epoch": 1.592,
|
||||||
|
"grad_norm": 33.37752151489258,
|
||||||
|
"learning_rate": 2.730670898658255e-06,
|
||||||
|
"logits/chosen": -0.8269758479064379,
|
||||||
|
"logits/rejected": -0.6836210629893789,
|
||||||
|
"logps/chosen": -157.39115772247314,
|
||||||
|
"logps/rejected": -160.14211597442628,
|
||||||
|
"loss": 0.33512029647827146,
|
||||||
|
"mean_token_accuracy": 0.4874211989343166,
|
||||||
|
"num_tokens": 489806.0,
|
||||||
|
"rewards/accuracies": 0.8875,
|
||||||
|
"rewards/chosen": 0.32783279549330474,
|
||||||
|
"rewards/margins": 1.7466752111911774,
|
||||||
|
"rewards/rejected": -1.4188424333930016,
|
||||||
|
"step": 100
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 1.592,
|
||||||
|
"eval_entropy": 2.1636376428604125,
|
||||||
|
"eval_logits/chosen": -0.8136097147525514,
|
||||||
|
"eval_logits/rejected": -0.7600604176985916,
|
||||||
|
"eval_logps/chosen": -167.09777592468262,
|
||||||
|
"eval_logps/rejected": -159.95339458465577,
|
||||||
|
"eval_loss": 0.7532036304473877,
|
||||||
|
"eval_mean_token_accuracy": 0.4550632082223892,
|
||||||
|
"eval_num_tokens": 489806.0,
|
||||||
|
"eval_rewards/accuracies": 0.531,
|
||||||
|
"eval_rewards/chosen": -0.5649967772588134,
|
||||||
|
"eval_rewards/margins": 0.1147943808734417,
|
||||||
|
"eval_rewards/rejected": -0.679791161686182,
|
||||||
|
"eval_runtime": 204.516,
|
||||||
|
"eval_samples_per_second": 4.89,
|
||||||
|
"eval_steps_per_second": 1.222,
|
||||||
|
"step": 100
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"logging_steps": 5,
|
||||||
|
"max_steps": 189,
|
||||||
|
"num_input_tokens_seen": 0,
|
||||||
|
"num_train_epochs": 3,
|
||||||
|
"save_steps": 100,
|
||||||
|
"stateful_callbacks": {
|
||||||
|
"TrainerControl": {
|
||||||
|
"args": {
|
||||||
|
"should_epoch_stop": false,
|
||||||
|
"should_evaluate": false,
|
||||||
|
"should_log": false,
|
||||||
|
"should_save": true,
|
||||||
|
"should_training_stop": false
|
||||||
|
},
|
||||||
|
"attributes": {}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"total_flos": 5237912290590720.0,
|
||||||
|
"train_batch_size": 4,
|
||||||
|
"trial_name": null,
|
||||||
|
"trial_params": null
|
||||||
|
}
|
||||||
3
checkpoint-100/training_args.bin
Normal file
3
checkpoint-100/training_args.bin
Normal file
@@ -0,0 +1,3 @@
|
|||||||
|
version https://git-lfs.github.com/spec/v1
|
||||||
|
oid sha256:f43b17fdc84ada6be616c284883b7a7a3bafe3a5ce12291db551a4ebc9781c8c
|
||||||
|
size 5841
|
||||||
33
checkpoint-189/config.json
Normal file
33
checkpoint-189/config.json
Normal file
@@ -0,0 +1,33 @@
|
|||||||
|
{
|
||||||
|
"architectures": [
|
||||||
|
"GPTNeoXForCausalLM"
|
||||||
|
],
|
||||||
|
"attention_bias": true,
|
||||||
|
"attention_dropout": 0.0,
|
||||||
|
"bos_token_id": 0,
|
||||||
|
"classifier_dropout": 0.1,
|
||||||
|
"dtype": "float32",
|
||||||
|
"eos_token_id": 0,
|
||||||
|
"hidden_act": "gelu",
|
||||||
|
"hidden_dropout": 0.0,
|
||||||
|
"hidden_size": 1024,
|
||||||
|
"initializer_range": 0.02,
|
||||||
|
"intermediate_size": 4096,
|
||||||
|
"is_decoder": false,
|
||||||
|
"layer_norm_eps": 1e-05,
|
||||||
|
"max_position_embeddings": 2048,
|
||||||
|
"model_type": "gpt_neox",
|
||||||
|
"num_attention_heads": 16,
|
||||||
|
"num_hidden_layers": 24,
|
||||||
|
"pad_token_id": 1,
|
||||||
|
"rope_parameters": {
|
||||||
|
"partial_rotary_factor": 0.25,
|
||||||
|
"rope_theta": 10000,
|
||||||
|
"rope_type": "default"
|
||||||
|
},
|
||||||
|
"tie_word_embeddings": false,
|
||||||
|
"transformers_version": "5.13.0",
|
||||||
|
"use_cache": false,
|
||||||
|
"use_parallel_residual": true,
|
||||||
|
"vocab_size": 50304
|
||||||
|
}
|
||||||
10
checkpoint-189/generation_config.json
Normal file
10
checkpoint-189/generation_config.json
Normal file
@@ -0,0 +1,10 @@
|
|||||||
|
{
|
||||||
|
"_from_model_config": true,
|
||||||
|
"bos_token_id": 0,
|
||||||
|
"eos_token_id": [
|
||||||
|
0
|
||||||
|
],
|
||||||
|
"pad_token_id": 1,
|
||||||
|
"transformers_version": "5.13.0",
|
||||||
|
"use_cache": true
|
||||||
|
}
|
||||||
3
checkpoint-189/model.safetensors
Normal file
3
checkpoint-189/model.safetensors
Normal file
@@ -0,0 +1,3 @@
|
|||||||
|
version https://git-lfs.github.com/spec/v1
|
||||||
|
oid sha256:fec90aa0de29c34e29db4101176e95b38fe7658c8e510641bbfaaa260772c186
|
||||||
|
size 1621370224
|
||||||
3
checkpoint-189/optimizer.pt
Normal file
3
checkpoint-189/optimizer.pt
Normal file
@@ -0,0 +1,3 @@
|
|||||||
|
version https://git-lfs.github.com/spec/v1
|
||||||
|
oid sha256:7cc2547a90db3b3b4cb704805a3dcab07e3243c92483664580ab532b30080ca1
|
||||||
|
size 3242916491
|
||||||
3
checkpoint-189/rng_state.pth
Normal file
3
checkpoint-189/rng_state.pth
Normal file
@@ -0,0 +1,3 @@
|
|||||||
|
version https://git-lfs.github.com/spec/v1
|
||||||
|
oid sha256:fdc37bbd2e979f041dfbbb004a5c74bab6cdda159cb18116df728588515a9ef6
|
||||||
|
size 14645
|
||||||
3
checkpoint-189/scaler.pt
Normal file
3
checkpoint-189/scaler.pt
Normal file
@@ -0,0 +1,3 @@
|
|||||||
|
version https://git-lfs.github.com/spec/v1
|
||||||
|
oid sha256:49f6bef68f1a79850e9c094ae1ada2d209bb00bda2944425e211ef8d4b2d7e85
|
||||||
|
size 1383
|
||||||
3
checkpoint-189/scheduler.pt
Normal file
3
checkpoint-189/scheduler.pt
Normal file
@@ -0,0 +1,3 @@
|
|||||||
|
version https://git-lfs.github.com/spec/v1
|
||||||
|
oid sha256:315777cdc55fe9505686ba500aff0de04b86af28c9ef728058b9b40c43f12bd4
|
||||||
|
size 1465
|
||||||
250557
checkpoint-189/tokenizer.json
Normal file
250557
checkpoint-189/tokenizer.json
Normal file
File diff suppressed because it is too large
Load Diff
14
checkpoint-189/tokenizer_config.json
Normal file
14
checkpoint-189/tokenizer_config.json
Normal file
@@ -0,0 +1,14 @@
|
|||||||
|
{
|
||||||
|
"add_prefix_space": false,
|
||||||
|
"backend": "tokenizers",
|
||||||
|
"bos_token": "<|endoftext|>",
|
||||||
|
"eos_token": "<|endoftext|>",
|
||||||
|
"errors": "replace",
|
||||||
|
"is_local": false,
|
||||||
|
"local_files_only": false,
|
||||||
|
"model_max_length": 1000000000000000019884624838656,
|
||||||
|
"pad_token": "<|padding|>",
|
||||||
|
"tokenizer_class": "GPTNeoXTokenizer",
|
||||||
|
"trim_offsets": true,
|
||||||
|
"unk_token": "<|endoftext|>"
|
||||||
|
}
|
||||||
795
checkpoint-189/trainer_state.json
Normal file
795
checkpoint-189/trainer_state.json
Normal file
@@ -0,0 +1,795 @@
|
|||||||
|
{
|
||||||
|
"best_global_step": null,
|
||||||
|
"best_metric": null,
|
||||||
|
"best_model_checkpoint": null,
|
||||||
|
"epoch": 3.0,
|
||||||
|
"eval_steps": 50,
|
||||||
|
"global_step": 189,
|
||||||
|
"is_hyper_param_search": false,
|
||||||
|
"is_local_process_zero": true,
|
||||||
|
"is_world_process_zero": true,
|
||||||
|
"log_history": [
|
||||||
|
{
|
||||||
|
"entropy": 2.555654156208038,
|
||||||
|
"epoch": 0.08,
|
||||||
|
"grad_norm": 35.3809700012207,
|
||||||
|
"learning_rate": 2.8571428571428573e-06,
|
||||||
|
"logits/chosen": 0.4110352631849171,
|
||||||
|
"logits/rejected": 0.4300117739250047,
|
||||||
|
"logps/chosen": -157.81439809799195,
|
||||||
|
"logps/rejected": -150.4315113067627,
|
||||||
|
"loss": 0.6903757572174072,
|
||||||
|
"mean_token_accuracy": 0.45491820499300956,
|
||||||
|
"num_tokens": 63280.0,
|
||||||
|
"rewards/accuracies": 0.54375,
|
||||||
|
"rewards/chosen": 0.018655257727368736,
|
||||||
|
"rewards/margins": 0.006082923361100256,
|
||||||
|
"rewards/rejected": 0.012572333973366767,
|
||||||
|
"step": 5
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"entropy": 2.5844572484493256,
|
||||||
|
"epoch": 0.16,
|
||||||
|
"grad_norm": 47.283817291259766,
|
||||||
|
"learning_rate": 4.984280524733107e-06,
|
||||||
|
"logits/chosen": 0.2089756903127898,
|
||||||
|
"logits/rejected": 0.3923359114864852,
|
||||||
|
"logps/chosen": -160.2436939239502,
|
||||||
|
"logps/rejected": -154.7825647354126,
|
||||||
|
"loss": 0.7087126731872558,
|
||||||
|
"mean_token_accuracy": 0.4676915630698204,
|
||||||
|
"num_tokens": 128751.0,
|
||||||
|
"rewards/accuracies": 0.46875,
|
||||||
|
"rewards/chosen": 0.026459120609797538,
|
||||||
|
"rewards/margins": -0.018515753094106913,
|
||||||
|
"rewards/rejected": 0.04497487433254719,
|
||||||
|
"step": 10
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"entropy": 2.4940366342663767,
|
||||||
|
"epoch": 0.24,
|
||||||
|
"grad_norm": 48.20133972167969,
|
||||||
|
"learning_rate": 4.809698831278217e-06,
|
||||||
|
"logits/chosen": 0.12422276096059184,
|
||||||
|
"logits/rejected": 0.18387425940175633,
|
||||||
|
"logps/chosen": -151.31794481277467,
|
||||||
|
"logps/rejected": -163.5261468887329,
|
||||||
|
"loss": 0.6959157466888428,
|
||||||
|
"mean_token_accuracy": 0.47235521450638773,
|
||||||
|
"num_tokens": 194142.0,
|
||||||
|
"rewards/accuracies": 0.46875,
|
||||||
|
"rewards/chosen": -0.059232240472920236,
|
||||||
|
"rewards/margins": 0.021671066922135652,
|
||||||
|
"rewards/rejected": -0.0809033086989075,
|
||||||
|
"step": 15
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"entropy": 2.5471011281013487,
|
||||||
|
"epoch": 0.32,
|
||||||
|
"grad_norm": 42.294281005859375,
|
||||||
|
"learning_rate": 4.454578706170075e-06,
|
||||||
|
"logits/chosen": -0.030133471365837695,
|
||||||
|
"logits/rejected": 0.08451241878365641,
|
||||||
|
"logps/chosen": -135.41445970535278,
|
||||||
|
"logps/rejected": -135.83921966552734,
|
||||||
|
"loss": 0.6954083442687988,
|
||||||
|
"mean_token_accuracy": 0.4549576163291931,
|
||||||
|
"num_tokens": 254871.0,
|
||||||
|
"rewards/accuracies": 0.5375,
|
||||||
|
"rewards/chosen": -0.18437330080196263,
|
||||||
|
"rewards/margins": 0.07001338726840914,
|
||||||
|
"rewards/rejected": -0.2543866881169379,
|
||||||
|
"step": 20
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"entropy": 2.4626115679740908,
|
||||||
|
"epoch": 0.4,
|
||||||
|
"grad_norm": 53.21565628051758,
|
||||||
|
"learning_rate": 3.946678240449515e-06,
|
||||||
|
"logits/chosen": 0.035210923156354317,
|
||||||
|
"logits/rejected": 0.03131556176291432,
|
||||||
|
"logps/chosen": -147.02334337234498,
|
||||||
|
"logps/rejected": -139.0251603126526,
|
||||||
|
"loss": 0.7144542694091797,
|
||||||
|
"mean_token_accuracy": 0.4505583204329014,
|
||||||
|
"num_tokens": 317752.0,
|
||||||
|
"rewards/accuracies": 0.55,
|
||||||
|
"rewards/chosen": -0.3056703276000917,
|
||||||
|
"rewards/margins": 0.030420188792049885,
|
||||||
|
"rewards/rejected": -0.33609051471576096,
|
||||||
|
"step": 25
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"entropy": 2.418185669183731,
|
||||||
|
"epoch": 0.48,
|
||||||
|
"grad_norm": 50.63364791870117,
|
||||||
|
"learning_rate": 3.3256976548879183e-06,
|
||||||
|
"logits/chosen": -0.0063577626599184795,
|
||||||
|
"logits/rejected": 0.0660789555792278,
|
||||||
|
"logps/chosen": -151.3002492904663,
|
||||||
|
"logps/rejected": -150.41238498687744,
|
||||||
|
"loss": 0.6920054435729981,
|
||||||
|
"mean_token_accuracy": 0.445500810444355,
|
||||||
|
"num_tokens": 387452.0,
|
||||||
|
"rewards/accuracies": 0.55625,
|
||||||
|
"rewards/chosen": -0.2962045251857489,
|
||||||
|
"rewards/margins": 0.08783619347959756,
|
||||||
|
"rewards/rejected": -0.38404072066769,
|
||||||
|
"step": 30
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"entropy": 2.42596016228199,
|
||||||
|
"epoch": 0.56,
|
||||||
|
"grad_norm": 47.31837844848633,
|
||||||
|
"learning_rate": 2.6401761180929798e-06,
|
||||||
|
"logits/chosen": -0.04082900087497934,
|
||||||
|
"logits/rejected": 0.015806176234233794,
|
||||||
|
"logps/chosen": -182.25081090927125,
|
||||||
|
"logps/rejected": -175.18973140716554,
|
||||||
|
"loss": 0.6894076824188232,
|
||||||
|
"mean_token_accuracy": 0.46841963976621626,
|
||||||
|
"num_tokens": 454873.0,
|
||||||
|
"rewards/accuracies": 0.55625,
|
||||||
|
"rewards/chosen": -0.30482858400791885,
|
||||||
|
"rewards/margins": 0.07829219745472074,
|
||||||
|
"rewards/rejected": -0.38312078421004114,
|
||||||
|
"step": 35
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"entropy": 2.421294018626213,
|
||||||
|
"epoch": 0.64,
|
||||||
|
"grad_norm": 47.72428894042969,
|
||||||
|
"learning_rate": 1.9436976651092143e-06,
|
||||||
|
"logits/chosen": 0.04491766159482387,
|
||||||
|
"logits/rejected": 0.15141158319763592,
|
||||||
|
"logps/chosen": -160.55729579925537,
|
||||||
|
"logps/rejected": -161.91398558616638,
|
||||||
|
"loss": 0.7262070178985596,
|
||||||
|
"mean_token_accuracy": 0.4616738587617874,
|
||||||
|
"num_tokens": 518527.0,
|
||||||
|
"rewards/accuracies": 0.51875,
|
||||||
|
"rewards/chosen": -0.2627937063574791,
|
||||||
|
"rewards/margins": 0.037597383465617897,
|
||||||
|
"rewards/rejected": -0.30039109364151956,
|
||||||
|
"step": 40
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"entropy": 2.3493838876485826,
|
||||||
|
"epoch": 0.72,
|
||||||
|
"grad_norm": 41.70436096191406,
|
||||||
|
"learning_rate": 1.2907027822369006e-06,
|
||||||
|
"logits/chosen": 0.06111884454416239,
|
||||||
|
"logits/rejected": 0.1077471313778626,
|
||||||
|
"logps/chosen": -155.73900985717773,
|
||||||
|
"logps/rejected": -162.23499221801757,
|
||||||
|
"loss": 0.6567366123199463,
|
||||||
|
"mean_token_accuracy": 0.47065152823925016,
|
||||||
|
"num_tokens": 586496.0,
|
||||||
|
"rewards/accuracies": 0.5875,
|
||||||
|
"rewards/chosen": -0.15574295134283603,
|
||||||
|
"rewards/margins": 0.16263325293548406,
|
||||||
|
"rewards/rejected": -0.31837619938887657,
|
||||||
|
"step": 45
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"entropy": 2.3660752564668655,
|
||||||
|
"epoch": 0.8,
|
||||||
|
"grad_norm": 46.26735305786133,
|
||||||
|
"learning_rate": 7.322330470336314e-07,
|
||||||
|
"logits/chosen": 0.04797605992282007,
|
||||||
|
"logits/rejected": 0.137719508348569,
|
||||||
|
"logps/chosen": -165.56337423324584,
|
||||||
|
"logps/rejected": -165.80605907440184,
|
||||||
|
"loss": 0.6880872249603271,
|
||||||
|
"mean_token_accuracy": 0.4618512228131294,
|
||||||
|
"num_tokens": 659345.0,
|
||||||
|
"rewards/accuracies": 0.51875,
|
||||||
|
"rewards/chosen": -0.13928775684908032,
|
||||||
|
"rewards/margins": 0.09893286619335413,
|
||||||
|
"rewards/rejected": -0.238220629747957,
|
||||||
|
"step": 50
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.8,
|
||||||
|
"eval_entropy": 2.4305524230003357,
|
||||||
|
"eval_logits/chosen": 0.060791561005096514,
|
||||||
|
"eval_logits/rejected": 0.1157722840661163,
|
||||||
|
"eval_logps/chosen": -163.22620516967774,
|
||||||
|
"eval_logps/rejected": -155.69468914794922,
|
||||||
|
"eval_loss": 0.699152946472168,
|
||||||
|
"eval_mean_token_accuracy": 0.4591814261674881,
|
||||||
|
"eval_num_tokens": 659345.0,
|
||||||
|
"eval_rewards/accuracies": 0.523,
|
||||||
|
"eval_rewards/chosen": -0.17783968531899155,
|
||||||
|
"eval_rewards/margins": 0.07608089716732502,
|
||||||
|
"eval_rewards/rejected": -0.2539205798432231,
|
||||||
|
"eval_runtime": 204.6975,
|
||||||
|
"eval_samples_per_second": 4.885,
|
||||||
|
"eval_steps_per_second": 1.221,
|
||||||
|
"step": 50
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"entropy": 2.3940737187862395,
|
||||||
|
"epoch": 0.88,
|
||||||
|
"grad_norm": 48.0695915222168,
|
||||||
|
"learning_rate": 3.119414452281158e-07,
|
||||||
|
"logits/chosen": 0.07112344903443206,
|
||||||
|
"logits/rejected": 0.19535127089682655,
|
||||||
|
"logps/chosen": -150.24068050384523,
|
||||||
|
"logps/rejected": -145.48797540664674,
|
||||||
|
"loss": 0.697087287902832,
|
||||||
|
"mean_token_accuracy": 0.4701634831726551,
|
||||||
|
"num_tokens": 731475.0,
|
||||||
|
"rewards/accuracies": 0.56875,
|
||||||
|
"rewards/chosen": -0.14963094238191843,
|
||||||
|
"rewards/margins": 0.08578414842486382,
|
||||||
|
"rewards/rejected": -0.2354150922037661,
|
||||||
|
"step": 55
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"entropy": 2.357463392615318,
|
||||||
|
"epoch": 0.96,
|
||||||
|
"grad_norm": 48.10399627685547,
|
||||||
|
"learning_rate": 6.268021954544095e-08,
|
||||||
|
"logits/chosen": 0.01995007363434443,
|
||||||
|
"logits/rejected": 0.16687794622622143,
|
||||||
|
"logps/chosen": -158.93942432403566,
|
||||||
|
"logps/rejected": -166.66946430206298,
|
||||||
|
"loss": 0.6701779842376709,
|
||||||
|
"mean_token_accuracy": 0.4689980365335941,
|
||||||
|
"num_tokens": 797757.0,
|
||||||
|
"rewards/accuracies": 0.5375,
|
||||||
|
"rewards/chosen": -0.1631853505037725,
|
||||||
|
"rewards/margins": 0.13551276298239828,
|
||||||
|
"rewards/rejected": -0.2986981191672385,
|
||||||
|
"step": 60
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 1.0,
|
||||||
|
"eval_entropy": 2.431350471973419,
|
||||||
|
"eval_logits/chosen": 0.058027013522972216,
|
||||||
|
"eval_logits/rejected": 0.11281541237759458,
|
||||||
|
"eval_logps/chosen": -163.3362089691162,
|
||||||
|
"eval_logps/rejected": -155.8263526611328,
|
||||||
|
"eval_loss": 0.6985306143760681,
|
||||||
|
"eval_mean_token_accuracy": 0.45935997557640074,
|
||||||
|
"eval_num_tokens": 829784.0,
|
||||||
|
"eval_rewards/accuracies": 0.529,
|
||||||
|
"eval_rewards/chosen": -0.188840083822608,
|
||||||
|
"eval_rewards/margins": 0.07824680726230145,
|
||||||
|
"eval_rewards/rejected": -0.26708688925206664,
|
||||||
|
"eval_runtime": 204.726,
|
||||||
|
"eval_samples_per_second": 4.885,
|
||||||
|
"eval_steps_per_second": 1.221,
|
||||||
|
"step": 63
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"entropy": 2.402912601828575,
|
||||||
|
"epoch": 1.032,
|
||||||
|
"grad_norm": 32.69980239868164,
|
||||||
|
"learning_rate": 4.184239109116393e-06,
|
||||||
|
"logits/chosen": 0.05448504114962113,
|
||||||
|
"logits/rejected": 0.09865282891809334,
|
||||||
|
"logps/chosen": -146.60927033424377,
|
||||||
|
"logps/rejected": -172.2928113937378,
|
||||||
|
"loss": 0.3366573750972748,
|
||||||
|
"mean_token_accuracy": 0.4740424118936062,
|
||||||
|
"num_tokens": 27180.0,
|
||||||
|
"rewards/accuracies": 0.859375,
|
||||||
|
"rewards/chosen": 0.5033185752108693,
|
||||||
|
"rewards/margins": 1.529684765264392,
|
||||||
|
"rewards/rejected": -1.0263662077486515,
|
||||||
|
"step": 65
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"entropy": 2.3992972910404204,
|
||||||
|
"epoch": 1.112,
|
||||||
|
"grad_norm": 17.76884651184082,
|
||||||
|
"learning_rate": 4.006586590948141e-06,
|
||||||
|
"logits/chosen": 0.0026862717521520412,
|
||||||
|
"logits/rejected": 0.13353262495170462,
|
||||||
|
"logps/chosen": -153.0885206222534,
|
||||||
|
"logps/rejected": -156.96604356765747,
|
||||||
|
"loss": 0.3243739604949951,
|
||||||
|
"mean_token_accuracy": 0.47915840819478034,
|
||||||
|
"num_tokens": 86611.0,
|
||||||
|
"rewards/accuracies": 0.8625,
|
||||||
|
"rewards/chosen": 0.4796073419041932,
|
||||||
|
"rewards/margins": 1.6072665184736252,
|
||||||
|
"rewards/rejected": -1.1276591904461384,
|
||||||
|
"step": 70
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"entropy": 2.3780136823654177,
|
||||||
|
"epoch": 1.192,
|
||||||
|
"grad_norm": 22.48740005493164,
|
||||||
|
"learning_rate": 3.81608040719339e-06,
|
||||||
|
"logits/chosen": -0.11707000613882948,
|
||||||
|
"logits/rejected": -0.06640926333257077,
|
||||||
|
"logps/chosen": -153.8634229660034,
|
||||||
|
"logps/rejected": -163.86640148162843,
|
||||||
|
"loss": 0.35583436489105225,
|
||||||
|
"mean_token_accuracy": 0.4692669503390789,
|
||||||
|
"num_tokens": 153056.0,
|
||||||
|
"rewards/accuracies": 0.83125,
|
||||||
|
"rewards/chosen": 0.23908968223258853,
|
||||||
|
"rewards/margins": 1.5137944724410772,
|
||||||
|
"rewards/rejected": -1.2747048027813435,
|
||||||
|
"step": 75
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"entropy": 2.310171273350716,
|
||||||
|
"epoch": 1.272,
|
||||||
|
"grad_norm": 19.922534942626953,
|
||||||
|
"learning_rate": 3.6143458894413463e-06,
|
||||||
|
"logits/chosen": -0.3582654699758253,
|
||||||
|
"logits/rejected": -0.2914110555931979,
|
||||||
|
"logps/chosen": -131.75142288208008,
|
||||||
|
"logps/rejected": -160.25545768737794,
|
||||||
|
"loss": 0.33200573921203613,
|
||||||
|
"mean_token_accuracy": 0.4669229932129383,
|
||||||
|
"num_tokens": 215920.0,
|
||||||
|
"rewards/accuracies": 0.8875,
|
||||||
|
"rewards/chosen": 0.15461167003959417,
|
||||||
|
"rewards/margins": 1.571430729702115,
|
||||||
|
"rewards/rejected": -1.4168190509080887,
|
||||||
|
"step": 80
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"entropy": 2.211333890259266,
|
||||||
|
"epoch": 1.3519999999999999,
|
||||||
|
"grad_norm": 26.273008346557617,
|
||||||
|
"learning_rate": 3.403104165467883e-06,
|
||||||
|
"logits/chosen": -0.4359116041083054,
|
||||||
|
"logits/rejected": -0.3527264562702821,
|
||||||
|
"logps/chosen": -140.573939037323,
|
||||||
|
"logps/rejected": -165.2755883216858,
|
||||||
|
"loss": 0.3167553186416626,
|
||||||
|
"mean_token_accuracy": 0.49101626574993135,
|
||||||
|
"num_tokens": 287127.0,
|
||||||
|
"rewards/accuracies": 0.86875,
|
||||||
|
"rewards/chosen": 0.22212664941325783,
|
||||||
|
"rewards/margins": 1.8355195850133896,
|
||||||
|
"rewards/rejected": -1.6133929178118707,
|
||||||
|
"step": 85
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"entropy": 2.284930866956711,
|
||||||
|
"epoch": 1.432,
|
||||||
|
"grad_norm": 23.699182510375977,
|
||||||
|
"learning_rate": 3.184157475180208e-06,
|
||||||
|
"logits/chosen": -0.49555035845434314,
|
||||||
|
"logits/rejected": -0.4411274731208847,
|
||||||
|
"logps/chosen": -159.97724952697754,
|
||||||
|
"logps/rejected": -158.18697261810303,
|
||||||
|
"loss": 0.31732380390167236,
|
||||||
|
"mean_token_accuracy": 0.46273171752691267,
|
||||||
|
"num_tokens": 354068.0,
|
||||||
|
"rewards/accuracies": 0.9,
|
||||||
|
"rewards/chosen": 0.2169143119826913,
|
||||||
|
"rewards/margins": 1.6415132291615009,
|
||||||
|
"rewards/rejected": -1.4245989315211773,
|
||||||
|
"step": 90
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"entropy": 2.185201385617256,
|
||||||
|
"epoch": 1.512,
|
||||||
|
"grad_norm": 19.721302032470703,
|
||||||
|
"learning_rate": 2.9593737945414264e-06,
|
||||||
|
"logits/chosen": -0.6350667553105876,
|
||||||
|
"logits/rejected": -0.6234260760266512,
|
||||||
|
"logps/chosen": -140.15507984161377,
|
||||||
|
"logps/rejected": -165.66675672531127,
|
||||||
|
"loss": 0.3336572885513306,
|
||||||
|
"mean_token_accuracy": 0.46677022203803065,
|
||||||
|
"num_tokens": 421426.0,
|
||||||
|
"rewards/accuracies": 0.8375,
|
||||||
|
"rewards/chosen": 0.1235144454985857,
|
||||||
|
"rewards/margins": 1.6951029762625693,
|
||||||
|
"rewards/rejected": -1.5715885123237967,
|
||||||
|
"step": 95
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"entropy": 2.116330775618553,
|
||||||
|
"epoch": 1.592,
|
||||||
|
"grad_norm": 33.37752151489258,
|
||||||
|
"learning_rate": 2.730670898658255e-06,
|
||||||
|
"logits/chosen": -0.8269758479064379,
|
||||||
|
"logits/rejected": -0.6836210629893789,
|
||||||
|
"logps/chosen": -157.39115772247314,
|
||||||
|
"logps/rejected": -160.14211597442628,
|
||||||
|
"loss": 0.33512029647827146,
|
||||||
|
"mean_token_accuracy": 0.4874211989343166,
|
||||||
|
"num_tokens": 489806.0,
|
||||||
|
"rewards/accuracies": 0.8875,
|
||||||
|
"rewards/chosen": 0.32783279549330474,
|
||||||
|
"rewards/margins": 1.7466752111911774,
|
||||||
|
"rewards/rejected": -1.4188424333930016,
|
||||||
|
"step": 100
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 1.592,
|
||||||
|
"eval_entropy": 2.1636376428604125,
|
||||||
|
"eval_logits/chosen": -0.8136097147525514,
|
||||||
|
"eval_logits/rejected": -0.7600604176985916,
|
||||||
|
"eval_logps/chosen": -167.09777592468262,
|
||||||
|
"eval_logps/rejected": -159.95339458465577,
|
||||||
|
"eval_loss": 0.7532036304473877,
|
||||||
|
"eval_mean_token_accuracy": 0.4550632082223892,
|
||||||
|
"eval_num_tokens": 489806.0,
|
||||||
|
"eval_rewards/accuracies": 0.531,
|
||||||
|
"eval_rewards/chosen": -0.5649967772588134,
|
||||||
|
"eval_rewards/margins": 0.1147943808734417,
|
||||||
|
"eval_rewards/rejected": -0.679791161686182,
|
||||||
|
"eval_runtime": 204.516,
|
||||||
|
"eval_samples_per_second": 4.89,
|
||||||
|
"eval_steps_per_second": 1.222,
|
||||||
|
"step": 100
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"entropy": 2.1376398682594298,
|
||||||
|
"epoch": 1.6720000000000002,
|
||||||
|
"grad_norm": 26.6827335357666,
|
||||||
|
"learning_rate": 2.5e-06,
|
||||||
|
"logits/chosen": -0.817711521022764,
|
||||||
|
"logits/rejected": -0.7502540023514636,
|
||||||
|
"logps/chosen": -166.91539459228517,
|
||||||
|
"logps/rejected": -191.0643165588379,
|
||||||
|
"loss": 0.27990891933441164,
|
||||||
|
"mean_token_accuracy": 0.4785373486578465,
|
||||||
|
"num_tokens": 67320.0,
|
||||||
|
"rewards/accuracies": 0.9375,
|
||||||
|
"rewards/chosen": 0.4409050527960062,
|
||||||
|
"rewards/margins": 2.258123181760311,
|
||||||
|
"rewards/rejected": -1.8172180943191052,
|
||||||
|
"step": 105
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"entropy": 2.059819537401199,
|
||||||
|
"epoch": 1.752,
|
||||||
|
"grad_norm": 24.675567626953125,
|
||||||
|
"learning_rate": 2.269329101341745e-06,
|
||||||
|
"logits/chosen": -0.9049248824738976,
|
||||||
|
"logits/rejected": -0.8284383619431989,
|
||||||
|
"logps/chosen": -159.55292291641234,
|
||||||
|
"logps/rejected": -177.96484870910643,
|
||||||
|
"loss": 0.28675858974456786,
|
||||||
|
"mean_token_accuracy": 0.4875997893512249,
|
||||||
|
"num_tokens": 132140.0,
|
||||||
|
"rewards/accuracies": 0.925,
|
||||||
|
"rewards/chosen": 0.37930642515420915,
|
||||||
|
"rewards/margins": 1.9427161723375321,
|
||||||
|
"rewards/rejected": -1.5634097740054131,
|
||||||
|
"step": 110
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"entropy": 2.037626123428345,
|
||||||
|
"epoch": 1.8319999999999999,
|
||||||
|
"grad_norm": 21.444379806518555,
|
||||||
|
"learning_rate": 2.040626205458574e-06,
|
||||||
|
"logits/chosen": -1.010849450295792,
|
||||||
|
"logits/rejected": -0.8793864952798423,
|
||||||
|
"logps/chosen": -151.68567123413087,
|
||||||
|
"logps/rejected": -171.75303897857665,
|
||||||
|
"loss": 0.31492087841033933,
|
||||||
|
"mean_token_accuracy": 0.48049742802977563,
|
||||||
|
"num_tokens": 201358.0,
|
||||||
|
"rewards/accuracies": 0.8875,
|
||||||
|
"rewards/chosen": 0.22566586136817932,
|
||||||
|
"rewards/margins": 1.7971878163516521,
|
||||||
|
"rewards/rejected": -1.5715219512581826,
|
||||||
|
"step": 115
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"entropy": 2.0510205030441284,
|
||||||
|
"epoch": 1.912,
|
||||||
|
"grad_norm": 23.6982364654541,
|
||||||
|
"learning_rate": 1.8158425248197931e-06,
|
||||||
|
"logits/chosen": -0.9916353547588466,
|
||||||
|
"logits/rejected": -0.919767900124741,
|
||||||
|
"logps/chosen": -154.16109809875488,
|
||||||
|
"logps/rejected": -166.89081039428712,
|
||||||
|
"loss": 0.3056188106536865,
|
||||||
|
"mean_token_accuracy": 0.4722383268177509,
|
||||||
|
"num_tokens": 264721.0,
|
||||||
|
"rewards/accuracies": 0.86875,
|
||||||
|
"rewards/chosen": 0.2006932887248695,
|
||||||
|
"rewards/margins": 1.8468371406197548,
|
||||||
|
"rewards/rejected": -1.646143864095211,
|
||||||
|
"step": 120
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"entropy": 2.093482795357704,
|
||||||
|
"epoch": 1.992,
|
||||||
|
"grad_norm": 22.745805740356445,
|
||||||
|
"learning_rate": 1.5968958345321178e-06,
|
||||||
|
"logits/chosen": -1.0079677581836826,
|
||||||
|
"logits/rejected": -0.9303146415112165,
|
||||||
|
"logps/chosen": -148.35024213790894,
|
||||||
|
"logps/rejected": -164.5137134552002,
|
||||||
|
"loss": 0.3322633981704712,
|
||||||
|
"mean_token_accuracy": 0.4708759218454361,
|
||||||
|
"num_tokens": 334037.0,
|
||||||
|
"rewards/accuracies": 0.85625,
|
||||||
|
"rewards/chosen": 0.11466053407639265,
|
||||||
|
"rewards/margins": 1.6517433917149902,
|
||||||
|
"rewards/rejected": -1.5370828822255134,
|
||||||
|
"step": 125
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"entropy": 2.0945689843760595,
|
||||||
|
"epoch": 2.064,
|
||||||
|
"grad_norm": 10.98022174835205,
|
||||||
|
"learning_rate": 1.3856541105586545e-06,
|
||||||
|
"logits/chosen": -0.9942241856066296,
|
||||||
|
"logits/rejected": -0.9094278407875448,
|
||||||
|
"logps/chosen": -140.99659379323325,
|
||||||
|
"logps/rejected": -170.61128277248807,
|
||||||
|
"loss": 0.1790446400642395,
|
||||||
|
"mean_token_accuracy": 0.49307963914341396,
|
||||||
|
"num_tokens": 390815.0,
|
||||||
|
"rewards/accuracies": 0.9583333333333334,
|
||||||
|
"rewards/chosen": 0.6758623485349946,
|
||||||
|
"rewards/margins": 2.894124209880829,
|
||||||
|
"rewards/rejected": -2.218261867761612,
|
||||||
|
"step": 130
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"entropy": 2.0529139041900635,
|
||||||
|
"epoch": 2.144,
|
||||||
|
"grad_norm": 10.028863906860352,
|
||||||
|
"learning_rate": 1.1839195928066101e-06,
|
||||||
|
"logits/chosen": -1.0788186523834689,
|
||||||
|
"logits/rejected": -0.986635527035572,
|
||||||
|
"logps/chosen": -145.5455855369568,
|
||||||
|
"logps/rejected": -172.8044556617737,
|
||||||
|
"loss": 0.12198150157928467,
|
||||||
|
"mean_token_accuracy": 0.49735122323036196,
|
||||||
|
"num_tokens": 457256.0,
|
||||||
|
"rewards/accuracies": 0.975,
|
||||||
|
"rewards/chosen": 0.6768257994204759,
|
||||||
|
"rewards/margins": 3.0055406376719476,
|
||||||
|
"rewards/rejected": -2.3287148490548133,
|
||||||
|
"step": 135
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"entropy": 2.014680004119873,
|
||||||
|
"epoch": 2.224,
|
||||||
|
"grad_norm": 6.895854473114014,
|
||||||
|
"learning_rate": 9.934134090518593e-07,
|
||||||
|
"logits/chosen": -1.1362325910319455,
|
||||||
|
"logits/rejected": -1.002460196026479,
|
||||||
|
"logps/chosen": -141.88460431098937,
|
||||||
|
"logps/rejected": -159.63373432159423,
|
||||||
|
"loss": 0.14334821701049805,
|
||||||
|
"mean_token_accuracy": 0.48404414057731626,
|
||||||
|
"num_tokens": 519814.0,
|
||||||
|
"rewards/accuracies": 0.95,
|
||||||
|
"rewards/chosen": 0.5280852797441185,
|
||||||
|
"rewards/margins": 2.968889817595482,
|
||||||
|
"rewards/rejected": -2.4408045306801798,
|
||||||
|
"step": 140
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"entropy": 1.9911637425422668,
|
||||||
|
"epoch": 2.304,
|
||||||
|
"grad_norm": 10.900385856628418,
|
||||||
|
"learning_rate": 8.157608908836071e-07,
|
||||||
|
"logits/chosen": -1.240284110529506,
|
||||||
|
"logits/rejected": -1.2038341543285929,
|
||||||
|
"logps/chosen": -149.32998390197753,
|
||||||
|
"logps/rejected": -179.72271556854247,
|
||||||
|
"loss": 0.11892527341842651,
|
||||||
|
"mean_token_accuracy": 0.48943382054567336,
|
||||||
|
"num_tokens": 583559.0,
|
||||||
|
"rewards/accuracies": 0.975,
|
||||||
|
"rewards/chosen": 0.5228322677314281,
|
||||||
|
"rewards/margins": 3.2153540045022964,
|
||||||
|
"rewards/rejected": -2.6925217270851136,
|
||||||
|
"step": 145
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"entropy": 2.0109089225530625,
|
||||||
|
"epoch": 2.384,
|
||||||
|
"grad_norm": 7.562666893005371,
|
||||||
|
"learning_rate": 6.524777069483526e-07,
|
||||||
|
"logits/chosen": -1.362585639730851,
|
||||||
|
"logits/rejected": -1.3277683060910546,
|
||||||
|
"logps/chosen": -152.4582862854004,
|
||||||
|
"logps/rejected": -190.31861305236816,
|
||||||
|
"loss": 0.1071157455444336,
|
||||||
|
"mean_token_accuracy": 0.47375648468732834,
|
||||||
|
"num_tokens": 655280.0,
|
||||||
|
"rewards/accuracies": 0.95,
|
||||||
|
"rewards/chosen": 0.5572835557162762,
|
||||||
|
"rewards/margins": 3.580769953131676,
|
||||||
|
"rewards/rejected": -3.023486441373825,
|
||||||
|
"step": 150
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 2.384,
|
||||||
|
"eval_entropy": 1.9984020476341247,
|
||||||
|
"eval_logits/chosen": -1.3867654543550485,
|
||||||
|
"eval_logits/rejected": -1.3457294671252336,
|
||||||
|
"eval_logps/chosen": -172.3030450744629,
|
||||||
|
"eval_logps/rejected": -165.6812739868164,
|
||||||
|
"eval_loss": 0.7880498766899109,
|
||||||
|
"eval_mean_token_accuracy": 0.451040216088295,
|
||||||
|
"eval_num_tokens": 655280.0,
|
||||||
|
"eval_rewards/accuracies": 0.54,
|
||||||
|
"eval_rewards/chosen": -1.0855237049460411,
|
||||||
|
"eval_rewards/margins": 0.1670554026067257,
|
||||||
|
"eval_rewards/rejected": -1.25257910490036,
|
||||||
|
"eval_runtime": 203.3449,
|
||||||
|
"eval_samples_per_second": 4.918,
|
||||||
|
"eval_steps_per_second": 1.229,
|
||||||
|
"step": 150
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"entropy": 1.898311933875084,
|
||||||
|
"epoch": 2.464,
|
||||||
|
"grad_norm": 11.360427856445312,
|
||||||
|
"learning_rate": 5.049569317994013e-07,
|
||||||
|
"logits/chosen": -1.4353495696293097,
|
||||||
|
"logits/rejected": -1.3758243692230172,
|
||||||
|
"logps/chosen": -141.70478200912476,
|
||||||
|
"logps/rejected": -200.1671936035156,
|
||||||
|
"loss": 0.09740959405899048,
|
||||||
|
"mean_token_accuracy": 0.4936802975833416,
|
||||||
|
"num_tokens": 724110.0,
|
||||||
|
"rewards/accuracies": 0.96875,
|
||||||
|
"rewards/chosen": 0.4263562995940447,
|
||||||
|
"rewards/margins": 3.59362336397171,
|
||||||
|
"rewards/rejected": -3.167267033457756,
|
||||||
|
"step": 155
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"entropy": 1.939386433362961,
|
||||||
|
"epoch": 2.544,
|
||||||
|
"grad_norm": 8.742157936096191,
|
||||||
|
"learning_rate": 3.7445716067596506e-07,
|
||||||
|
"logits/chosen": -1.4550549474115178,
|
||||||
|
"logits/rejected": -1.3327657359030232,
|
||||||
|
"logps/chosen": -132.01672496795655,
|
||||||
|
"logps/rejected": -159.5728401184082,
|
||||||
|
"loss": 0.10442214012145996,
|
||||||
|
"mean_token_accuracy": 0.4859817087650299,
|
||||||
|
"num_tokens": 787960.0,
|
||||||
|
"rewards/accuracies": 0.98125,
|
||||||
|
"rewards/chosen": 0.42371090203523637,
|
||||||
|
"rewards/margins": 3.1901376724243162,
|
||||||
|
"rewards/rejected": -2.7664267748594282,
|
||||||
|
"step": 160
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"entropy": 1.9298288524150848,
|
||||||
|
"epoch": 2.624,
|
||||||
|
"grad_norm": 6.683835029602051,
|
||||||
|
"learning_rate": 2.620917716123444e-07,
|
||||||
|
"logits/chosen": -1.5218572998191655,
|
||||||
|
"logits/rejected": -1.5028220084321073,
|
||||||
|
"logps/chosen": -146.9026556968689,
|
||||||
|
"logps/rejected": -182.4629114151001,
|
||||||
|
"loss": 0.11852377653121948,
|
||||||
|
"mean_token_accuracy": 0.5008164420723915,
|
||||||
|
"num_tokens": 854381.0,
|
||||||
|
"rewards/accuracies": 0.975,
|
||||||
|
"rewards/chosen": 0.340854674577713,
|
||||||
|
"rewards/margins": 3.3918783754110335,
|
||||||
|
"rewards/rejected": -3.051023706793785,
|
||||||
|
"step": 165
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"entropy": 1.9614687770605088,
|
||||||
|
"epoch": 2.7039999999999997,
|
||||||
|
"grad_norm": 8.654135704040527,
|
||||||
|
"learning_rate": 1.6881942648911077e-07,
|
||||||
|
"logits/chosen": -1.483899302043358,
|
||||||
|
"logits/rejected": -1.3984594015444671,
|
||||||
|
"logps/chosen": -165.00297966003419,
|
||||||
|
"logps/rejected": -223.27304649353027,
|
||||||
|
"loss": 0.10529448986053466,
|
||||||
|
"mean_token_accuracy": 0.4777096025645733,
|
||||||
|
"num_tokens": 922218.0,
|
||||||
|
"rewards/accuracies": 0.9625,
|
||||||
|
"rewards/chosen": 0.1868226437829435,
|
||||||
|
"rewards/margins": 3.6259077370166777,
|
||||||
|
"rewards/rejected": -3.439085066318512,
|
||||||
|
"step": 170
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"entropy": 1.947239425778389,
|
||||||
|
"epoch": 2.784,
|
||||||
|
"grad_norm": 8.482812881469727,
|
||||||
|
"learning_rate": 9.54358920679524e-08,
|
||||||
|
"logits/chosen": -1.5538378778015716,
|
||||||
|
"logits/rejected": -1.4374772630805093,
|
||||||
|
"logps/chosen": -167.4766321182251,
|
||||||
|
"logps/rejected": -185.17562103271484,
|
||||||
|
"loss": 0.10844930410385131,
|
||||||
|
"mean_token_accuracy": 0.48532758429646494,
|
||||||
|
"num_tokens": 989301.0,
|
||||||
|
"rewards/accuracies": 0.98125,
|
||||||
|
"rewards/chosen": 0.5600280098617076,
|
||||||
|
"rewards/margins": 3.5826878786087035,
|
||||||
|
"rewards/rejected": -3.0226599007844923,
|
||||||
|
"step": 175
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"entropy": 1.9497412502765656,
|
||||||
|
"epoch": 2.864,
|
||||||
|
"grad_norm": 10.068187713623047,
|
||||||
|
"learning_rate": 4.256725079024554e-08,
|
||||||
|
"logits/chosen": -1.5250048523452793,
|
||||||
|
"logits/rejected": -1.4418669213263424,
|
||||||
|
"logps/chosen": -154.15428905487062,
|
||||||
|
"logps/rejected": -163.1813151359558,
|
||||||
|
"loss": 0.13129655122756959,
|
||||||
|
"mean_token_accuracy": 0.4707398295402527,
|
||||||
|
"num_tokens": 1057139.0,
|
||||||
|
"rewards/accuracies": 0.9625,
|
||||||
|
"rewards/chosen": 0.4407920534722507,
|
||||||
|
"rewards/margins": 3.0125378251075743,
|
||||||
|
"rewards/rejected": -2.571745750308037,
|
||||||
|
"step": 180
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"entropy": 1.9340084820985795,
|
||||||
|
"epoch": 2.944,
|
||||||
|
"grad_norm": 9.49740219116211,
|
||||||
|
"learning_rate": 1.0664559262413831e-08,
|
||||||
|
"logits/chosen": -1.512207703275473,
|
||||||
|
"logits/rejected": -1.4001704843220182,
|
||||||
|
"logps/chosen": -157.45920839309693,
|
||||||
|
"logps/rejected": -180.69428844451903,
|
||||||
|
"loss": 0.09854952096939087,
|
||||||
|
"mean_token_accuracy": 0.4828268110752106,
|
||||||
|
"num_tokens": 1120692.0,
|
||||||
|
"rewards/accuracies": 0.96875,
|
||||||
|
"rewards/chosen": 0.4773704446852207,
|
||||||
|
"rewards/margins": 3.379485607147217,
|
||||||
|
"rewards/rejected": -2.9021151691675184,
|
||||||
|
"step": 185
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 3.0,
|
||||||
|
"eval_entropy": 1.9640341634750367,
|
||||||
|
"eval_logits/chosen": -1.5042375842896538,
|
||||||
|
"eval_logits/rejected": -1.4667329044378787,
|
||||||
|
"eval_logps/chosen": -173.94541304016113,
|
||||||
|
"eval_logps/rejected": -167.3245940551758,
|
||||||
|
"eval_loss": 0.8192874193191528,
|
||||||
|
"eval_mean_token_accuracy": 0.44932303857803346,
|
||||||
|
"eval_num_tokens": 1169762.0,
|
||||||
|
"eval_rewards/accuracies": 0.534,
|
||||||
|
"eval_rewards/chosen": -1.2497604206800461,
|
||||||
|
"eval_rewards/margins": 0.16715064680576325,
|
||||||
|
"eval_rewards/rejected": -1.4169110633730888,
|
||||||
|
"eval_runtime": 203.5561,
|
||||||
|
"eval_samples_per_second": 4.913,
|
||||||
|
"eval_steps_per_second": 1.228,
|
||||||
|
"step": 189
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"logging_steps": 5,
|
||||||
|
"max_steps": 189,
|
||||||
|
"num_input_tokens_seen": 0,
|
||||||
|
"num_train_epochs": 3,
|
||||||
|
"save_steps": 100,
|
||||||
|
"stateful_callbacks": {
|
||||||
|
"TrainerControl": {
|
||||||
|
"args": {
|
||||||
|
"should_epoch_stop": false,
|
||||||
|
"should_evaluate": false,
|
||||||
|
"should_log": false,
|
||||||
|
"should_save": true,
|
||||||
|
"should_training_stop": true
|
||||||
|
},
|
||||||
|
"attributes": {}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"total_flos": 9840760918671360.0,
|
||||||
|
"train_batch_size": 4,
|
||||||
|
"trial_name": null,
|
||||||
|
"trial_params": null
|
||||||
|
}
|
||||||
3
checkpoint-189/training_args.bin
Normal file
3
checkpoint-189/training_args.bin
Normal file
@@ -0,0 +1,3 @@
|
|||||||
|
version https://git-lfs.github.com/spec/v1
|
||||||
|
oid sha256:f43b17fdc84ada6be616c284883b7a7a3bafe3a5ce12291db551a4ebc9781c8c
|
||||||
|
size 5841
|
||||||
33
checkpoint-63/config.json
Normal file
33
checkpoint-63/config.json
Normal file
@@ -0,0 +1,33 @@
|
|||||||
|
{
|
||||||
|
"architectures": [
|
||||||
|
"GPTNeoXForCausalLM"
|
||||||
|
],
|
||||||
|
"attention_bias": true,
|
||||||
|
"attention_dropout": 0.0,
|
||||||
|
"bos_token_id": 0,
|
||||||
|
"classifier_dropout": 0.1,
|
||||||
|
"dtype": "float32",
|
||||||
|
"eos_token_id": 0,
|
||||||
|
"hidden_act": "gelu",
|
||||||
|
"hidden_dropout": 0.0,
|
||||||
|
"hidden_size": 1024,
|
||||||
|
"initializer_range": 0.02,
|
||||||
|
"intermediate_size": 4096,
|
||||||
|
"is_decoder": false,
|
||||||
|
"layer_norm_eps": 1e-05,
|
||||||
|
"max_position_embeddings": 2048,
|
||||||
|
"model_type": "gpt_neox",
|
||||||
|
"num_attention_heads": 16,
|
||||||
|
"num_hidden_layers": 24,
|
||||||
|
"pad_token_id": 1,
|
||||||
|
"rope_parameters": {
|
||||||
|
"partial_rotary_factor": 0.25,
|
||||||
|
"rope_theta": 10000,
|
||||||
|
"rope_type": "default"
|
||||||
|
},
|
||||||
|
"tie_word_embeddings": false,
|
||||||
|
"transformers_version": "5.13.0",
|
||||||
|
"use_cache": false,
|
||||||
|
"use_parallel_residual": true,
|
||||||
|
"vocab_size": 50304
|
||||||
|
}
|
||||||
10
checkpoint-63/generation_config.json
Normal file
10
checkpoint-63/generation_config.json
Normal file
@@ -0,0 +1,10 @@
|
|||||||
|
{
|
||||||
|
"_from_model_config": true,
|
||||||
|
"bos_token_id": 0,
|
||||||
|
"eos_token_id": [
|
||||||
|
0
|
||||||
|
],
|
||||||
|
"pad_token_id": 1,
|
||||||
|
"transformers_version": "5.13.0",
|
||||||
|
"use_cache": true
|
||||||
|
}
|
||||||
3
checkpoint-63/model.safetensors
Normal file
3
checkpoint-63/model.safetensors
Normal file
@@ -0,0 +1,3 @@
|
|||||||
|
version https://git-lfs.github.com/spec/v1
|
||||||
|
oid sha256:e0fc7603d000fae40859e8c2397447edbf0b90b4318888df42c3fa912f4285d4
|
||||||
|
size 1621370224
|
||||||
3
checkpoint-63/optimizer.pt
Normal file
3
checkpoint-63/optimizer.pt
Normal file
@@ -0,0 +1,3 @@
|
|||||||
|
version https://git-lfs.github.com/spec/v1
|
||||||
|
oid sha256:07577cf19a615723516ca2a380ef0d69c7ed060acc82810f5c1e9fdb8650093d
|
||||||
|
size 3242916491
|
||||||
3
checkpoint-63/rng_state.pth
Normal file
3
checkpoint-63/rng_state.pth
Normal file
@@ -0,0 +1,3 @@
|
|||||||
|
version https://git-lfs.github.com/spec/v1
|
||||||
|
oid sha256:f4a9f217e852f439efa6bd32fde98d6867f11aa6ea13ddc021ba10af6a0b0934
|
||||||
|
size 14645
|
||||||
3
checkpoint-63/scaler.pt
Normal file
3
checkpoint-63/scaler.pt
Normal file
@@ -0,0 +1,3 @@
|
|||||||
|
version https://git-lfs.github.com/spec/v1
|
||||||
|
oid sha256:7b09ff53dfdeb98feaa25ccade54f490eaebb7ca5b59d6e930f01eab95790c26
|
||||||
|
size 1383
|
||||||
3
checkpoint-63/scheduler.pt
Normal file
3
checkpoint-63/scheduler.pt
Normal file
@@ -0,0 +1,3 @@
|
|||||||
|
version https://git-lfs.github.com/spec/v1
|
||||||
|
oid sha256:8f144286c127833c5309618e8e24f8a84d423b58fb8de34fb79a94bb5b178ab9
|
||||||
|
size 1465
|
||||||
250557
checkpoint-63/tokenizer.json
Normal file
250557
checkpoint-63/tokenizer.json
Normal file
File diff suppressed because it is too large
Load Diff
14
checkpoint-63/tokenizer_config.json
Normal file
14
checkpoint-63/tokenizer_config.json
Normal file
@@ -0,0 +1,14 @@
|
|||||||
|
{
|
||||||
|
"add_prefix_space": false,
|
||||||
|
"backend": "tokenizers",
|
||||||
|
"bos_token": "<|endoftext|>",
|
||||||
|
"eos_token": "<|endoftext|>",
|
||||||
|
"errors": "replace",
|
||||||
|
"is_local": false,
|
||||||
|
"local_files_only": false,
|
||||||
|
"model_max_length": 1000000000000000019884624838656,
|
||||||
|
"pad_token": "<|padding|>",
|
||||||
|
"tokenizer_class": "GPTNeoXTokenizer",
|
||||||
|
"trim_offsets": true,
|
||||||
|
"unk_token": "<|endoftext|>"
|
||||||
|
}
|
||||||
288
checkpoint-63/trainer_state.json
Normal file
288
checkpoint-63/trainer_state.json
Normal file
@@ -0,0 +1,288 @@
|
|||||||
|
{
|
||||||
|
"best_global_step": null,
|
||||||
|
"best_metric": null,
|
||||||
|
"best_model_checkpoint": null,
|
||||||
|
"epoch": 1.0,
|
||||||
|
"eval_steps": 50,
|
||||||
|
"global_step": 63,
|
||||||
|
"is_hyper_param_search": false,
|
||||||
|
"is_local_process_zero": true,
|
||||||
|
"is_world_process_zero": true,
|
||||||
|
"log_history": [
|
||||||
|
{
|
||||||
|
"entropy": 2.555654156208038,
|
||||||
|
"epoch": 0.08,
|
||||||
|
"grad_norm": 35.3809700012207,
|
||||||
|
"learning_rate": 2.8571428571428573e-06,
|
||||||
|
"logits/chosen": 0.4110352631849171,
|
||||||
|
"logits/rejected": 0.4300117739250047,
|
||||||
|
"logps/chosen": -157.81439809799195,
|
||||||
|
"logps/rejected": -150.4315113067627,
|
||||||
|
"loss": 0.6903757572174072,
|
||||||
|
"mean_token_accuracy": 0.45491820499300956,
|
||||||
|
"num_tokens": 63280.0,
|
||||||
|
"rewards/accuracies": 0.54375,
|
||||||
|
"rewards/chosen": 0.018655257727368736,
|
||||||
|
"rewards/margins": 0.006082923361100256,
|
||||||
|
"rewards/rejected": 0.012572333973366767,
|
||||||
|
"step": 5
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"entropy": 2.5844572484493256,
|
||||||
|
"epoch": 0.16,
|
||||||
|
"grad_norm": 47.283817291259766,
|
||||||
|
"learning_rate": 4.984280524733107e-06,
|
||||||
|
"logits/chosen": 0.2089756903127898,
|
||||||
|
"logits/rejected": 0.3923359114864852,
|
||||||
|
"logps/chosen": -160.2436939239502,
|
||||||
|
"logps/rejected": -154.7825647354126,
|
||||||
|
"loss": 0.7087126731872558,
|
||||||
|
"mean_token_accuracy": 0.4676915630698204,
|
||||||
|
"num_tokens": 128751.0,
|
||||||
|
"rewards/accuracies": 0.46875,
|
||||||
|
"rewards/chosen": 0.026459120609797538,
|
||||||
|
"rewards/margins": -0.018515753094106913,
|
||||||
|
"rewards/rejected": 0.04497487433254719,
|
||||||
|
"step": 10
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"entropy": 2.4940366342663767,
|
||||||
|
"epoch": 0.24,
|
||||||
|
"grad_norm": 48.20133972167969,
|
||||||
|
"learning_rate": 4.809698831278217e-06,
|
||||||
|
"logits/chosen": 0.12422276096059184,
|
||||||
|
"logits/rejected": 0.18387425940175633,
|
||||||
|
"logps/chosen": -151.31794481277467,
|
||||||
|
"logps/rejected": -163.5261468887329,
|
||||||
|
"loss": 0.6959157466888428,
|
||||||
|
"mean_token_accuracy": 0.47235521450638773,
|
||||||
|
"num_tokens": 194142.0,
|
||||||
|
"rewards/accuracies": 0.46875,
|
||||||
|
"rewards/chosen": -0.059232240472920236,
|
||||||
|
"rewards/margins": 0.021671066922135652,
|
||||||
|
"rewards/rejected": -0.0809033086989075,
|
||||||
|
"step": 15
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"entropy": 2.5471011281013487,
|
||||||
|
"epoch": 0.32,
|
||||||
|
"grad_norm": 42.294281005859375,
|
||||||
|
"learning_rate": 4.454578706170075e-06,
|
||||||
|
"logits/chosen": -0.030133471365837695,
|
||||||
|
"logits/rejected": 0.08451241878365641,
|
||||||
|
"logps/chosen": -135.41445970535278,
|
||||||
|
"logps/rejected": -135.83921966552734,
|
||||||
|
"loss": 0.6954083442687988,
|
||||||
|
"mean_token_accuracy": 0.4549576163291931,
|
||||||
|
"num_tokens": 254871.0,
|
||||||
|
"rewards/accuracies": 0.5375,
|
||||||
|
"rewards/chosen": -0.18437330080196263,
|
||||||
|
"rewards/margins": 0.07001338726840914,
|
||||||
|
"rewards/rejected": -0.2543866881169379,
|
||||||
|
"step": 20
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"entropy": 2.4626115679740908,
|
||||||
|
"epoch": 0.4,
|
||||||
|
"grad_norm": 53.21565628051758,
|
||||||
|
"learning_rate": 3.946678240449515e-06,
|
||||||
|
"logits/chosen": 0.035210923156354317,
|
||||||
|
"logits/rejected": 0.03131556176291432,
|
||||||
|
"logps/chosen": -147.02334337234498,
|
||||||
|
"logps/rejected": -139.0251603126526,
|
||||||
|
"loss": 0.7144542694091797,
|
||||||
|
"mean_token_accuracy": 0.4505583204329014,
|
||||||
|
"num_tokens": 317752.0,
|
||||||
|
"rewards/accuracies": 0.55,
|
||||||
|
"rewards/chosen": -0.3056703276000917,
|
||||||
|
"rewards/margins": 0.030420188792049885,
|
||||||
|
"rewards/rejected": -0.33609051471576096,
|
||||||
|
"step": 25
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"entropy": 2.418185669183731,
|
||||||
|
"epoch": 0.48,
|
||||||
|
"grad_norm": 50.63364791870117,
|
||||||
|
"learning_rate": 3.3256976548879183e-06,
|
||||||
|
"logits/chosen": -0.0063577626599184795,
|
||||||
|
"logits/rejected": 0.0660789555792278,
|
||||||
|
"logps/chosen": -151.3002492904663,
|
||||||
|
"logps/rejected": -150.41238498687744,
|
||||||
|
"loss": 0.6920054435729981,
|
||||||
|
"mean_token_accuracy": 0.445500810444355,
|
||||||
|
"num_tokens": 387452.0,
|
||||||
|
"rewards/accuracies": 0.55625,
|
||||||
|
"rewards/chosen": -0.2962045251857489,
|
||||||
|
"rewards/margins": 0.08783619347959756,
|
||||||
|
"rewards/rejected": -0.38404072066769,
|
||||||
|
"step": 30
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"entropy": 2.42596016228199,
|
||||||
|
"epoch": 0.56,
|
||||||
|
"grad_norm": 47.31837844848633,
|
||||||
|
"learning_rate": 2.6401761180929798e-06,
|
||||||
|
"logits/chosen": -0.04082900087497934,
|
||||||
|
"logits/rejected": 0.015806176234233794,
|
||||||
|
"logps/chosen": -182.25081090927125,
|
||||||
|
"logps/rejected": -175.18973140716554,
|
||||||
|
"loss": 0.6894076824188232,
|
||||||
|
"mean_token_accuracy": 0.46841963976621626,
|
||||||
|
"num_tokens": 454873.0,
|
||||||
|
"rewards/accuracies": 0.55625,
|
||||||
|
"rewards/chosen": -0.30482858400791885,
|
||||||
|
"rewards/margins": 0.07829219745472074,
|
||||||
|
"rewards/rejected": -0.38312078421004114,
|
||||||
|
"step": 35
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"entropy": 2.421294018626213,
|
||||||
|
"epoch": 0.64,
|
||||||
|
"grad_norm": 47.72428894042969,
|
||||||
|
"learning_rate": 1.9436976651092143e-06,
|
||||||
|
"logits/chosen": 0.04491766159482387,
|
||||||
|
"logits/rejected": 0.15141158319763592,
|
||||||
|
"logps/chosen": -160.55729579925537,
|
||||||
|
"logps/rejected": -161.91398558616638,
|
||||||
|
"loss": 0.7262070178985596,
|
||||||
|
"mean_token_accuracy": 0.4616738587617874,
|
||||||
|
"num_tokens": 518527.0,
|
||||||
|
"rewards/accuracies": 0.51875,
|
||||||
|
"rewards/chosen": -0.2627937063574791,
|
||||||
|
"rewards/margins": 0.037597383465617897,
|
||||||
|
"rewards/rejected": -0.30039109364151956,
|
||||||
|
"step": 40
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"entropy": 2.3493838876485826,
|
||||||
|
"epoch": 0.72,
|
||||||
|
"grad_norm": 41.70436096191406,
|
||||||
|
"learning_rate": 1.2907027822369006e-06,
|
||||||
|
"logits/chosen": 0.06111884454416239,
|
||||||
|
"logits/rejected": 0.1077471313778626,
|
||||||
|
"logps/chosen": -155.73900985717773,
|
||||||
|
"logps/rejected": -162.23499221801757,
|
||||||
|
"loss": 0.6567366123199463,
|
||||||
|
"mean_token_accuracy": 0.47065152823925016,
|
||||||
|
"num_tokens": 586496.0,
|
||||||
|
"rewards/accuracies": 0.5875,
|
||||||
|
"rewards/chosen": -0.15574295134283603,
|
||||||
|
"rewards/margins": 0.16263325293548406,
|
||||||
|
"rewards/rejected": -0.31837619938887657,
|
||||||
|
"step": 45
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"entropy": 2.3660752564668655,
|
||||||
|
"epoch": 0.8,
|
||||||
|
"grad_norm": 46.26735305786133,
|
||||||
|
"learning_rate": 7.322330470336314e-07,
|
||||||
|
"logits/chosen": 0.04797605992282007,
|
||||||
|
"logits/rejected": 0.137719508348569,
|
||||||
|
"logps/chosen": -165.56337423324584,
|
||||||
|
"logps/rejected": -165.80605907440184,
|
||||||
|
"loss": 0.6880872249603271,
|
||||||
|
"mean_token_accuracy": 0.4618512228131294,
|
||||||
|
"num_tokens": 659345.0,
|
||||||
|
"rewards/accuracies": 0.51875,
|
||||||
|
"rewards/chosen": -0.13928775684908032,
|
||||||
|
"rewards/margins": 0.09893286619335413,
|
||||||
|
"rewards/rejected": -0.238220629747957,
|
||||||
|
"step": 50
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.8,
|
||||||
|
"eval_entropy": 2.4305524230003357,
|
||||||
|
"eval_logits/chosen": 0.060791561005096514,
|
||||||
|
"eval_logits/rejected": 0.1157722840661163,
|
||||||
|
"eval_logps/chosen": -163.22620516967774,
|
||||||
|
"eval_logps/rejected": -155.69468914794922,
|
||||||
|
"eval_loss": 0.699152946472168,
|
||||||
|
"eval_mean_token_accuracy": 0.4591814261674881,
|
||||||
|
"eval_num_tokens": 659345.0,
|
||||||
|
"eval_rewards/accuracies": 0.523,
|
||||||
|
"eval_rewards/chosen": -0.17783968531899155,
|
||||||
|
"eval_rewards/margins": 0.07608089716732502,
|
||||||
|
"eval_rewards/rejected": -0.2539205798432231,
|
||||||
|
"eval_runtime": 204.6975,
|
||||||
|
"eval_samples_per_second": 4.885,
|
||||||
|
"eval_steps_per_second": 1.221,
|
||||||
|
"step": 50
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"entropy": 2.3940737187862395,
|
||||||
|
"epoch": 0.88,
|
||||||
|
"grad_norm": 48.0695915222168,
|
||||||
|
"learning_rate": 3.119414452281158e-07,
|
||||||
|
"logits/chosen": 0.07112344903443206,
|
||||||
|
"logits/rejected": 0.19535127089682655,
|
||||||
|
"logps/chosen": -150.24068050384523,
|
||||||
|
"logps/rejected": -145.48797540664674,
|
||||||
|
"loss": 0.697087287902832,
|
||||||
|
"mean_token_accuracy": 0.4701634831726551,
|
||||||
|
"num_tokens": 731475.0,
|
||||||
|
"rewards/accuracies": 0.56875,
|
||||||
|
"rewards/chosen": -0.14963094238191843,
|
||||||
|
"rewards/margins": 0.08578414842486382,
|
||||||
|
"rewards/rejected": -0.2354150922037661,
|
||||||
|
"step": 55
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"entropy": 2.357463392615318,
|
||||||
|
"epoch": 0.96,
|
||||||
|
"grad_norm": 48.10399627685547,
|
||||||
|
"learning_rate": 6.268021954544095e-08,
|
||||||
|
"logits/chosen": 0.01995007363434443,
|
||||||
|
"logits/rejected": 0.16687794622622143,
|
||||||
|
"logps/chosen": -158.93942432403566,
|
||||||
|
"logps/rejected": -166.66946430206298,
|
||||||
|
"loss": 0.6701779842376709,
|
||||||
|
"mean_token_accuracy": 0.4689980365335941,
|
||||||
|
"num_tokens": 797757.0,
|
||||||
|
"rewards/accuracies": 0.5375,
|
||||||
|
"rewards/chosen": -0.1631853505037725,
|
||||||
|
"rewards/margins": 0.13551276298239828,
|
||||||
|
"rewards/rejected": -0.2986981191672385,
|
||||||
|
"step": 60
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 1.0,
|
||||||
|
"eval_entropy": 2.431350471973419,
|
||||||
|
"eval_logits/chosen": 0.058027013522972216,
|
||||||
|
"eval_logits/rejected": 0.11281541237759458,
|
||||||
|
"eval_logps/chosen": -163.3362089691162,
|
||||||
|
"eval_logps/rejected": -155.8263526611328,
|
||||||
|
"eval_loss": 0.6985306143760681,
|
||||||
|
"eval_mean_token_accuracy": 0.45935997557640074,
|
||||||
|
"eval_num_tokens": 829784.0,
|
||||||
|
"eval_rewards/accuracies": 0.529,
|
||||||
|
"eval_rewards/chosen": -0.188840083822608,
|
||||||
|
"eval_rewards/margins": 0.07824680726230145,
|
||||||
|
"eval_rewards/rejected": -0.26708688925206664,
|
||||||
|
"eval_runtime": 204.726,
|
||||||
|
"eval_samples_per_second": 4.885,
|
||||||
|
"eval_steps_per_second": 1.221,
|
||||||
|
"step": 63
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"logging_steps": 5,
|
||||||
|
"max_steps": 63,
|
||||||
|
"num_input_tokens_seen": 0,
|
||||||
|
"num_train_epochs": 1,
|
||||||
|
"save_steps": 100,
|
||||||
|
"stateful_callbacks": {
|
||||||
|
"TrainerControl": {
|
||||||
|
"args": {
|
||||||
|
"should_epoch_stop": false,
|
||||||
|
"should_evaluate": false,
|
||||||
|
"should_log": false,
|
||||||
|
"should_save": true,
|
||||||
|
"should_training_stop": true
|
||||||
|
},
|
||||||
|
"attributes": {}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"total_flos": 3267470732328960.0,
|
||||||
|
"train_batch_size": 4,
|
||||||
|
"trial_name": null,
|
||||||
|
"trial_params": null
|
||||||
|
}
|
||||||
3
checkpoint-63/training_args.bin
Normal file
3
checkpoint-63/training_args.bin
Normal file
@@ -0,0 +1,3 @@
|
|||||||
|
version https://git-lfs.github.com/spec/v1
|
||||||
|
oid sha256:e9c10bdc1b2e405fee996676bbed765046a6e06fe0fdaa6030fe444d7e535e6c
|
||||||
|
size 5841
|
||||||
33
config.json
Normal file
33
config.json
Normal file
@@ -0,0 +1,33 @@
|
|||||||
|
{
|
||||||
|
"architectures": [
|
||||||
|
"GPTNeoXForCausalLM"
|
||||||
|
],
|
||||||
|
"attention_bias": true,
|
||||||
|
"attention_dropout": 0.0,
|
||||||
|
"bos_token_id": 0,
|
||||||
|
"classifier_dropout": 0.1,
|
||||||
|
"dtype": "float32",
|
||||||
|
"eos_token_id": 0,
|
||||||
|
"hidden_act": "gelu",
|
||||||
|
"hidden_dropout": 0.0,
|
||||||
|
"hidden_size": 1024,
|
||||||
|
"initializer_range": 0.02,
|
||||||
|
"intermediate_size": 4096,
|
||||||
|
"is_decoder": false,
|
||||||
|
"layer_norm_eps": 1e-05,
|
||||||
|
"max_position_embeddings": 2048,
|
||||||
|
"model_type": "gpt_neox",
|
||||||
|
"num_attention_heads": 16,
|
||||||
|
"num_hidden_layers": 24,
|
||||||
|
"pad_token_id": 1,
|
||||||
|
"rope_parameters": {
|
||||||
|
"partial_rotary_factor": 0.25,
|
||||||
|
"rope_theta": 10000,
|
||||||
|
"rope_type": "default"
|
||||||
|
},
|
||||||
|
"tie_word_embeddings": false,
|
||||||
|
"transformers_version": "5.13.0",
|
||||||
|
"use_cache": false,
|
||||||
|
"use_parallel_residual": true,
|
||||||
|
"vocab_size": 50304
|
||||||
|
}
|
||||||
10
generation_config.json
Normal file
10
generation_config.json
Normal file
@@ -0,0 +1,10 @@
|
|||||||
|
{
|
||||||
|
"_from_model_config": true,
|
||||||
|
"bos_token_id": 0,
|
||||||
|
"eos_token_id": [
|
||||||
|
0
|
||||||
|
],
|
||||||
|
"pad_token_id": 1,
|
||||||
|
"transformers_version": "5.13.0",
|
||||||
|
"use_cache": true
|
||||||
|
}
|
||||||
763
log_history.json
Normal file
763
log_history.json
Normal file
@@ -0,0 +1,763 @@
|
|||||||
|
[
|
||||||
|
{
|
||||||
|
"entropy": 2.555654156208038,
|
||||||
|
"epoch": 0.08,
|
||||||
|
"grad_norm": 35.3809700012207,
|
||||||
|
"learning_rate": 2.8571428571428573e-06,
|
||||||
|
"logits/chosen": 0.4110352631849171,
|
||||||
|
"logits/rejected": 0.4300117739250047,
|
||||||
|
"logps/chosen": -157.81439809799195,
|
||||||
|
"logps/rejected": -150.4315113067627,
|
||||||
|
"loss": 0.6903757572174072,
|
||||||
|
"mean_token_accuracy": 0.45491820499300956,
|
||||||
|
"num_tokens": 63280.0,
|
||||||
|
"rewards/accuracies": 0.54375,
|
||||||
|
"rewards/chosen": 0.018655257727368736,
|
||||||
|
"rewards/margins": 0.006082923361100256,
|
||||||
|
"rewards/rejected": 0.012572333973366767,
|
||||||
|
"step": 5
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"entropy": 2.5844572484493256,
|
||||||
|
"epoch": 0.16,
|
||||||
|
"grad_norm": 47.283817291259766,
|
||||||
|
"learning_rate": 4.984280524733107e-06,
|
||||||
|
"logits/chosen": 0.2089756903127898,
|
||||||
|
"logits/rejected": 0.3923359114864852,
|
||||||
|
"logps/chosen": -160.2436939239502,
|
||||||
|
"logps/rejected": -154.7825647354126,
|
||||||
|
"loss": 0.7087126731872558,
|
||||||
|
"mean_token_accuracy": 0.4676915630698204,
|
||||||
|
"num_tokens": 128751.0,
|
||||||
|
"rewards/accuracies": 0.46875,
|
||||||
|
"rewards/chosen": 0.026459120609797538,
|
||||||
|
"rewards/margins": -0.018515753094106913,
|
||||||
|
"rewards/rejected": 0.04497487433254719,
|
||||||
|
"step": 10
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"entropy": 2.4940366342663767,
|
||||||
|
"epoch": 0.24,
|
||||||
|
"grad_norm": 48.20133972167969,
|
||||||
|
"learning_rate": 4.809698831278217e-06,
|
||||||
|
"logits/chosen": 0.12422276096059184,
|
||||||
|
"logits/rejected": 0.18387425940175633,
|
||||||
|
"logps/chosen": -151.31794481277467,
|
||||||
|
"logps/rejected": -163.5261468887329,
|
||||||
|
"loss": 0.6959157466888428,
|
||||||
|
"mean_token_accuracy": 0.47235521450638773,
|
||||||
|
"num_tokens": 194142.0,
|
||||||
|
"rewards/accuracies": 0.46875,
|
||||||
|
"rewards/chosen": -0.059232240472920236,
|
||||||
|
"rewards/margins": 0.021671066922135652,
|
||||||
|
"rewards/rejected": -0.0809033086989075,
|
||||||
|
"step": 15
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"entropy": 2.5471011281013487,
|
||||||
|
"epoch": 0.32,
|
||||||
|
"grad_norm": 42.294281005859375,
|
||||||
|
"learning_rate": 4.454578706170075e-06,
|
||||||
|
"logits/chosen": -0.030133471365837695,
|
||||||
|
"logits/rejected": 0.08451241878365641,
|
||||||
|
"logps/chosen": -135.41445970535278,
|
||||||
|
"logps/rejected": -135.83921966552734,
|
||||||
|
"loss": 0.6954083442687988,
|
||||||
|
"mean_token_accuracy": 0.4549576163291931,
|
||||||
|
"num_tokens": 254871.0,
|
||||||
|
"rewards/accuracies": 0.5375,
|
||||||
|
"rewards/chosen": -0.18437330080196263,
|
||||||
|
"rewards/margins": 0.07001338726840914,
|
||||||
|
"rewards/rejected": -0.2543866881169379,
|
||||||
|
"step": 20
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"entropy": 2.4626115679740908,
|
||||||
|
"epoch": 0.4,
|
||||||
|
"grad_norm": 53.21565628051758,
|
||||||
|
"learning_rate": 3.946678240449515e-06,
|
||||||
|
"logits/chosen": 0.035210923156354317,
|
||||||
|
"logits/rejected": 0.03131556176291432,
|
||||||
|
"logps/chosen": -147.02334337234498,
|
||||||
|
"logps/rejected": -139.0251603126526,
|
||||||
|
"loss": 0.7144542694091797,
|
||||||
|
"mean_token_accuracy": 0.4505583204329014,
|
||||||
|
"num_tokens": 317752.0,
|
||||||
|
"rewards/accuracies": 0.55,
|
||||||
|
"rewards/chosen": -0.3056703276000917,
|
||||||
|
"rewards/margins": 0.030420188792049885,
|
||||||
|
"rewards/rejected": -0.33609051471576096,
|
||||||
|
"step": 25
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"entropy": 2.418185669183731,
|
||||||
|
"epoch": 0.48,
|
||||||
|
"grad_norm": 50.63364791870117,
|
||||||
|
"learning_rate": 3.3256976548879183e-06,
|
||||||
|
"logits/chosen": -0.0063577626599184795,
|
||||||
|
"logits/rejected": 0.0660789555792278,
|
||||||
|
"logps/chosen": -151.3002492904663,
|
||||||
|
"logps/rejected": -150.41238498687744,
|
||||||
|
"loss": 0.6920054435729981,
|
||||||
|
"mean_token_accuracy": 0.445500810444355,
|
||||||
|
"num_tokens": 387452.0,
|
||||||
|
"rewards/accuracies": 0.55625,
|
||||||
|
"rewards/chosen": -0.2962045251857489,
|
||||||
|
"rewards/margins": 0.08783619347959756,
|
||||||
|
"rewards/rejected": -0.38404072066769,
|
||||||
|
"step": 30
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"entropy": 2.42596016228199,
|
||||||
|
"epoch": 0.56,
|
||||||
|
"grad_norm": 47.31837844848633,
|
||||||
|
"learning_rate": 2.6401761180929798e-06,
|
||||||
|
"logits/chosen": -0.04082900087497934,
|
||||||
|
"logits/rejected": 0.015806176234233794,
|
||||||
|
"logps/chosen": -182.25081090927125,
|
||||||
|
"logps/rejected": -175.18973140716554,
|
||||||
|
"loss": 0.6894076824188232,
|
||||||
|
"mean_token_accuracy": 0.46841963976621626,
|
||||||
|
"num_tokens": 454873.0,
|
||||||
|
"rewards/accuracies": 0.55625,
|
||||||
|
"rewards/chosen": -0.30482858400791885,
|
||||||
|
"rewards/margins": 0.07829219745472074,
|
||||||
|
"rewards/rejected": -0.38312078421004114,
|
||||||
|
"step": 35
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"entropy": 2.421294018626213,
|
||||||
|
"epoch": 0.64,
|
||||||
|
"grad_norm": 47.72428894042969,
|
||||||
|
"learning_rate": 1.9436976651092143e-06,
|
||||||
|
"logits/chosen": 0.04491766159482387,
|
||||||
|
"logits/rejected": 0.15141158319763592,
|
||||||
|
"logps/chosen": -160.55729579925537,
|
||||||
|
"logps/rejected": -161.91398558616638,
|
||||||
|
"loss": 0.7262070178985596,
|
||||||
|
"mean_token_accuracy": 0.4616738587617874,
|
||||||
|
"num_tokens": 518527.0,
|
||||||
|
"rewards/accuracies": 0.51875,
|
||||||
|
"rewards/chosen": -0.2627937063574791,
|
||||||
|
"rewards/margins": 0.037597383465617897,
|
||||||
|
"rewards/rejected": -0.30039109364151956,
|
||||||
|
"step": 40
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"entropy": 2.3493838876485826,
|
||||||
|
"epoch": 0.72,
|
||||||
|
"grad_norm": 41.70436096191406,
|
||||||
|
"learning_rate": 1.2907027822369006e-06,
|
||||||
|
"logits/chosen": 0.06111884454416239,
|
||||||
|
"logits/rejected": 0.1077471313778626,
|
||||||
|
"logps/chosen": -155.73900985717773,
|
||||||
|
"logps/rejected": -162.23499221801757,
|
||||||
|
"loss": 0.6567366123199463,
|
||||||
|
"mean_token_accuracy": 0.47065152823925016,
|
||||||
|
"num_tokens": 586496.0,
|
||||||
|
"rewards/accuracies": 0.5875,
|
||||||
|
"rewards/chosen": -0.15574295134283603,
|
||||||
|
"rewards/margins": 0.16263325293548406,
|
||||||
|
"rewards/rejected": -0.31837619938887657,
|
||||||
|
"step": 45
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"entropy": 2.3660752564668655,
|
||||||
|
"epoch": 0.8,
|
||||||
|
"grad_norm": 46.26735305786133,
|
||||||
|
"learning_rate": 7.322330470336314e-07,
|
||||||
|
"logits/chosen": 0.04797605992282007,
|
||||||
|
"logits/rejected": 0.137719508348569,
|
||||||
|
"logps/chosen": -165.56337423324584,
|
||||||
|
"logps/rejected": -165.80605907440184,
|
||||||
|
"loss": 0.6880872249603271,
|
||||||
|
"mean_token_accuracy": 0.4618512228131294,
|
||||||
|
"num_tokens": 659345.0,
|
||||||
|
"rewards/accuracies": 0.51875,
|
||||||
|
"rewards/chosen": -0.13928775684908032,
|
||||||
|
"rewards/margins": 0.09893286619335413,
|
||||||
|
"rewards/rejected": -0.238220629747957,
|
||||||
|
"step": 50
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.8,
|
||||||
|
"eval_entropy": 2.4305524230003357,
|
||||||
|
"eval_logits/chosen": 0.060791561005096514,
|
||||||
|
"eval_logits/rejected": 0.1157722840661163,
|
||||||
|
"eval_logps/chosen": -163.22620516967774,
|
||||||
|
"eval_logps/rejected": -155.69468914794922,
|
||||||
|
"eval_loss": 0.699152946472168,
|
||||||
|
"eval_mean_token_accuracy": 0.4591814261674881,
|
||||||
|
"eval_num_tokens": 659345.0,
|
||||||
|
"eval_rewards/accuracies": 0.523,
|
||||||
|
"eval_rewards/chosen": -0.17783968531899155,
|
||||||
|
"eval_rewards/margins": 0.07608089716732502,
|
||||||
|
"eval_rewards/rejected": -0.2539205798432231,
|
||||||
|
"eval_runtime": 204.6975,
|
||||||
|
"eval_samples_per_second": 4.885,
|
||||||
|
"eval_steps_per_second": 1.221,
|
||||||
|
"step": 50
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"entropy": 2.3940737187862395,
|
||||||
|
"epoch": 0.88,
|
||||||
|
"grad_norm": 48.0695915222168,
|
||||||
|
"learning_rate": 3.119414452281158e-07,
|
||||||
|
"logits/chosen": 0.07112344903443206,
|
||||||
|
"logits/rejected": 0.19535127089682655,
|
||||||
|
"logps/chosen": -150.24068050384523,
|
||||||
|
"logps/rejected": -145.48797540664674,
|
||||||
|
"loss": 0.697087287902832,
|
||||||
|
"mean_token_accuracy": 0.4701634831726551,
|
||||||
|
"num_tokens": 731475.0,
|
||||||
|
"rewards/accuracies": 0.56875,
|
||||||
|
"rewards/chosen": -0.14963094238191843,
|
||||||
|
"rewards/margins": 0.08578414842486382,
|
||||||
|
"rewards/rejected": -0.2354150922037661,
|
||||||
|
"step": 55
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"entropy": 2.357463392615318,
|
||||||
|
"epoch": 0.96,
|
||||||
|
"grad_norm": 48.10399627685547,
|
||||||
|
"learning_rate": 6.268021954544095e-08,
|
||||||
|
"logits/chosen": 0.01995007363434443,
|
||||||
|
"logits/rejected": 0.16687794622622143,
|
||||||
|
"logps/chosen": -158.93942432403566,
|
||||||
|
"logps/rejected": -166.66946430206298,
|
||||||
|
"loss": 0.6701779842376709,
|
||||||
|
"mean_token_accuracy": 0.4689980365335941,
|
||||||
|
"num_tokens": 797757.0,
|
||||||
|
"rewards/accuracies": 0.5375,
|
||||||
|
"rewards/chosen": -0.1631853505037725,
|
||||||
|
"rewards/margins": 0.13551276298239828,
|
||||||
|
"rewards/rejected": -0.2986981191672385,
|
||||||
|
"step": 60
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 1.0,
|
||||||
|
"eval_entropy": 2.431350471973419,
|
||||||
|
"eval_logits/chosen": 0.058027013522972216,
|
||||||
|
"eval_logits/rejected": 0.11281541237759458,
|
||||||
|
"eval_logps/chosen": -163.3362089691162,
|
||||||
|
"eval_logps/rejected": -155.8263526611328,
|
||||||
|
"eval_loss": 0.6985306143760681,
|
||||||
|
"eval_mean_token_accuracy": 0.45935997557640074,
|
||||||
|
"eval_num_tokens": 829784.0,
|
||||||
|
"eval_rewards/accuracies": 0.529,
|
||||||
|
"eval_rewards/chosen": -0.188840083822608,
|
||||||
|
"eval_rewards/margins": 0.07824680726230145,
|
||||||
|
"eval_rewards/rejected": -0.26708688925206664,
|
||||||
|
"eval_runtime": 204.726,
|
||||||
|
"eval_samples_per_second": 4.885,
|
||||||
|
"eval_steps_per_second": 1.221,
|
||||||
|
"step": 63
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"entropy": 2.402912601828575,
|
||||||
|
"epoch": 1.032,
|
||||||
|
"grad_norm": 32.69980239868164,
|
||||||
|
"learning_rate": 4.184239109116393e-06,
|
||||||
|
"logits/chosen": 0.05448504114962113,
|
||||||
|
"logits/rejected": 0.09865282891809334,
|
||||||
|
"logps/chosen": -146.60927033424377,
|
||||||
|
"logps/rejected": -172.2928113937378,
|
||||||
|
"loss": 0.3366573750972748,
|
||||||
|
"mean_token_accuracy": 0.4740424118936062,
|
||||||
|
"num_tokens": 27180.0,
|
||||||
|
"rewards/accuracies": 0.859375,
|
||||||
|
"rewards/chosen": 0.5033185752108693,
|
||||||
|
"rewards/margins": 1.529684765264392,
|
||||||
|
"rewards/rejected": -1.0263662077486515,
|
||||||
|
"step": 65
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"entropy": 2.3992972910404204,
|
||||||
|
"epoch": 1.112,
|
||||||
|
"grad_norm": 17.76884651184082,
|
||||||
|
"learning_rate": 4.006586590948141e-06,
|
||||||
|
"logits/chosen": 0.0026862717521520412,
|
||||||
|
"logits/rejected": 0.13353262495170462,
|
||||||
|
"logps/chosen": -153.0885206222534,
|
||||||
|
"logps/rejected": -156.96604356765747,
|
||||||
|
"loss": 0.3243739604949951,
|
||||||
|
"mean_token_accuracy": 0.47915840819478034,
|
||||||
|
"num_tokens": 86611.0,
|
||||||
|
"rewards/accuracies": 0.8625,
|
||||||
|
"rewards/chosen": 0.4796073419041932,
|
||||||
|
"rewards/margins": 1.6072665184736252,
|
||||||
|
"rewards/rejected": -1.1276591904461384,
|
||||||
|
"step": 70
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"entropy": 2.3780136823654177,
|
||||||
|
"epoch": 1.192,
|
||||||
|
"grad_norm": 22.48740005493164,
|
||||||
|
"learning_rate": 3.81608040719339e-06,
|
||||||
|
"logits/chosen": -0.11707000613882948,
|
||||||
|
"logits/rejected": -0.06640926333257077,
|
||||||
|
"logps/chosen": -153.8634229660034,
|
||||||
|
"logps/rejected": -163.86640148162843,
|
||||||
|
"loss": 0.35583436489105225,
|
||||||
|
"mean_token_accuracy": 0.4692669503390789,
|
||||||
|
"num_tokens": 153056.0,
|
||||||
|
"rewards/accuracies": 0.83125,
|
||||||
|
"rewards/chosen": 0.23908968223258853,
|
||||||
|
"rewards/margins": 1.5137944724410772,
|
||||||
|
"rewards/rejected": -1.2747048027813435,
|
||||||
|
"step": 75
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"entropy": 2.310171273350716,
|
||||||
|
"epoch": 1.272,
|
||||||
|
"grad_norm": 19.922534942626953,
|
||||||
|
"learning_rate": 3.6143458894413463e-06,
|
||||||
|
"logits/chosen": -0.3582654699758253,
|
||||||
|
"logits/rejected": -0.2914110555931979,
|
||||||
|
"logps/chosen": -131.75142288208008,
|
||||||
|
"logps/rejected": -160.25545768737794,
|
||||||
|
"loss": 0.33200573921203613,
|
||||||
|
"mean_token_accuracy": 0.4669229932129383,
|
||||||
|
"num_tokens": 215920.0,
|
||||||
|
"rewards/accuracies": 0.8875,
|
||||||
|
"rewards/chosen": 0.15461167003959417,
|
||||||
|
"rewards/margins": 1.571430729702115,
|
||||||
|
"rewards/rejected": -1.4168190509080887,
|
||||||
|
"step": 80
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"entropy": 2.211333890259266,
|
||||||
|
"epoch": 1.3519999999999999,
|
||||||
|
"grad_norm": 26.273008346557617,
|
||||||
|
"learning_rate": 3.403104165467883e-06,
|
||||||
|
"logits/chosen": -0.4359116041083054,
|
||||||
|
"logits/rejected": -0.3527264562702821,
|
||||||
|
"logps/chosen": -140.573939037323,
|
||||||
|
"logps/rejected": -165.2755883216858,
|
||||||
|
"loss": 0.3167553186416626,
|
||||||
|
"mean_token_accuracy": 0.49101626574993135,
|
||||||
|
"num_tokens": 287127.0,
|
||||||
|
"rewards/accuracies": 0.86875,
|
||||||
|
"rewards/chosen": 0.22212664941325783,
|
||||||
|
"rewards/margins": 1.8355195850133896,
|
||||||
|
"rewards/rejected": -1.6133929178118707,
|
||||||
|
"step": 85
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"entropy": 2.284930866956711,
|
||||||
|
"epoch": 1.432,
|
||||||
|
"grad_norm": 23.699182510375977,
|
||||||
|
"learning_rate": 3.184157475180208e-06,
|
||||||
|
"logits/chosen": -0.49555035845434314,
|
||||||
|
"logits/rejected": -0.4411274731208847,
|
||||||
|
"logps/chosen": -159.97724952697754,
|
||||||
|
"logps/rejected": -158.18697261810303,
|
||||||
|
"loss": 0.31732380390167236,
|
||||||
|
"mean_token_accuracy": 0.46273171752691267,
|
||||||
|
"num_tokens": 354068.0,
|
||||||
|
"rewards/accuracies": 0.9,
|
||||||
|
"rewards/chosen": 0.2169143119826913,
|
||||||
|
"rewards/margins": 1.6415132291615009,
|
||||||
|
"rewards/rejected": -1.4245989315211773,
|
||||||
|
"step": 90
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"entropy": 2.185201385617256,
|
||||||
|
"epoch": 1.512,
|
||||||
|
"grad_norm": 19.721302032470703,
|
||||||
|
"learning_rate": 2.9593737945414264e-06,
|
||||||
|
"logits/chosen": -0.6350667553105876,
|
||||||
|
"logits/rejected": -0.6234260760266512,
|
||||||
|
"logps/chosen": -140.15507984161377,
|
||||||
|
"logps/rejected": -165.66675672531127,
|
||||||
|
"loss": 0.3336572885513306,
|
||||||
|
"mean_token_accuracy": 0.46677022203803065,
|
||||||
|
"num_tokens": 421426.0,
|
||||||
|
"rewards/accuracies": 0.8375,
|
||||||
|
"rewards/chosen": 0.1235144454985857,
|
||||||
|
"rewards/margins": 1.6951029762625693,
|
||||||
|
"rewards/rejected": -1.5715885123237967,
|
||||||
|
"step": 95
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"entropy": 2.116330775618553,
|
||||||
|
"epoch": 1.592,
|
||||||
|
"grad_norm": 33.37752151489258,
|
||||||
|
"learning_rate": 2.730670898658255e-06,
|
||||||
|
"logits/chosen": -0.8269758479064379,
|
||||||
|
"logits/rejected": -0.6836210629893789,
|
||||||
|
"logps/chosen": -157.39115772247314,
|
||||||
|
"logps/rejected": -160.14211597442628,
|
||||||
|
"loss": 0.33512029647827146,
|
||||||
|
"mean_token_accuracy": 0.4874211989343166,
|
||||||
|
"num_tokens": 489806.0,
|
||||||
|
"rewards/accuracies": 0.8875,
|
||||||
|
"rewards/chosen": 0.32783279549330474,
|
||||||
|
"rewards/margins": 1.7466752111911774,
|
||||||
|
"rewards/rejected": -1.4188424333930016,
|
||||||
|
"step": 100
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 1.592,
|
||||||
|
"eval_entropy": 2.1636376428604125,
|
||||||
|
"eval_logits/chosen": -0.8136097147525514,
|
||||||
|
"eval_logits/rejected": -0.7600604176985916,
|
||||||
|
"eval_logps/chosen": -167.09777592468262,
|
||||||
|
"eval_logps/rejected": -159.95339458465577,
|
||||||
|
"eval_loss": 0.7532036304473877,
|
||||||
|
"eval_mean_token_accuracy": 0.4550632082223892,
|
||||||
|
"eval_num_tokens": 489806.0,
|
||||||
|
"eval_rewards/accuracies": 0.531,
|
||||||
|
"eval_rewards/chosen": -0.5649967772588134,
|
||||||
|
"eval_rewards/margins": 0.1147943808734417,
|
||||||
|
"eval_rewards/rejected": -0.679791161686182,
|
||||||
|
"eval_runtime": 204.516,
|
||||||
|
"eval_samples_per_second": 4.89,
|
||||||
|
"eval_steps_per_second": 1.222,
|
||||||
|
"step": 100
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"entropy": 2.1376398682594298,
|
||||||
|
"epoch": 1.6720000000000002,
|
||||||
|
"grad_norm": 26.6827335357666,
|
||||||
|
"learning_rate": 2.5e-06,
|
||||||
|
"logits/chosen": -0.817711521022764,
|
||||||
|
"logits/rejected": -0.7502540023514636,
|
||||||
|
"logps/chosen": -166.91539459228517,
|
||||||
|
"logps/rejected": -191.0643165588379,
|
||||||
|
"loss": 0.27990891933441164,
|
||||||
|
"mean_token_accuracy": 0.4785373486578465,
|
||||||
|
"num_tokens": 67320.0,
|
||||||
|
"rewards/accuracies": 0.9375,
|
||||||
|
"rewards/chosen": 0.4409050527960062,
|
||||||
|
"rewards/margins": 2.258123181760311,
|
||||||
|
"rewards/rejected": -1.8172180943191052,
|
||||||
|
"step": 105
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"entropy": 2.059819537401199,
|
||||||
|
"epoch": 1.752,
|
||||||
|
"grad_norm": 24.675567626953125,
|
||||||
|
"learning_rate": 2.269329101341745e-06,
|
||||||
|
"logits/chosen": -0.9049248824738976,
|
||||||
|
"logits/rejected": -0.8284383619431989,
|
||||||
|
"logps/chosen": -159.55292291641234,
|
||||||
|
"logps/rejected": -177.96484870910643,
|
||||||
|
"loss": 0.28675858974456786,
|
||||||
|
"mean_token_accuracy": 0.4875997893512249,
|
||||||
|
"num_tokens": 132140.0,
|
||||||
|
"rewards/accuracies": 0.925,
|
||||||
|
"rewards/chosen": 0.37930642515420915,
|
||||||
|
"rewards/margins": 1.9427161723375321,
|
||||||
|
"rewards/rejected": -1.5634097740054131,
|
||||||
|
"step": 110
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"entropy": 2.037626123428345,
|
||||||
|
"epoch": 1.8319999999999999,
|
||||||
|
"grad_norm": 21.444379806518555,
|
||||||
|
"learning_rate": 2.040626205458574e-06,
|
||||||
|
"logits/chosen": -1.010849450295792,
|
||||||
|
"logits/rejected": -0.8793864952798423,
|
||||||
|
"logps/chosen": -151.68567123413087,
|
||||||
|
"logps/rejected": -171.75303897857665,
|
||||||
|
"loss": 0.31492087841033933,
|
||||||
|
"mean_token_accuracy": 0.48049742802977563,
|
||||||
|
"num_tokens": 201358.0,
|
||||||
|
"rewards/accuracies": 0.8875,
|
||||||
|
"rewards/chosen": 0.22566586136817932,
|
||||||
|
"rewards/margins": 1.7971878163516521,
|
||||||
|
"rewards/rejected": -1.5715219512581826,
|
||||||
|
"step": 115
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"entropy": 2.0510205030441284,
|
||||||
|
"epoch": 1.912,
|
||||||
|
"grad_norm": 23.6982364654541,
|
||||||
|
"learning_rate": 1.8158425248197931e-06,
|
||||||
|
"logits/chosen": -0.9916353547588466,
|
||||||
|
"logits/rejected": -0.919767900124741,
|
||||||
|
"logps/chosen": -154.16109809875488,
|
||||||
|
"logps/rejected": -166.89081039428712,
|
||||||
|
"loss": 0.3056188106536865,
|
||||||
|
"mean_token_accuracy": 0.4722383268177509,
|
||||||
|
"num_tokens": 264721.0,
|
||||||
|
"rewards/accuracies": 0.86875,
|
||||||
|
"rewards/chosen": 0.2006932887248695,
|
||||||
|
"rewards/margins": 1.8468371406197548,
|
||||||
|
"rewards/rejected": -1.646143864095211,
|
||||||
|
"step": 120
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"entropy": 2.093482795357704,
|
||||||
|
"epoch": 1.992,
|
||||||
|
"grad_norm": 22.745805740356445,
|
||||||
|
"learning_rate": 1.5968958345321178e-06,
|
||||||
|
"logits/chosen": -1.0079677581836826,
|
||||||
|
"logits/rejected": -0.9303146415112165,
|
||||||
|
"logps/chosen": -148.35024213790894,
|
||||||
|
"logps/rejected": -164.5137134552002,
|
||||||
|
"loss": 0.3322633981704712,
|
||||||
|
"mean_token_accuracy": 0.4708759218454361,
|
||||||
|
"num_tokens": 334037.0,
|
||||||
|
"rewards/accuracies": 0.85625,
|
||||||
|
"rewards/chosen": 0.11466053407639265,
|
||||||
|
"rewards/margins": 1.6517433917149902,
|
||||||
|
"rewards/rejected": -1.5370828822255134,
|
||||||
|
"step": 125
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"entropy": 2.0945689843760595,
|
||||||
|
"epoch": 2.064,
|
||||||
|
"grad_norm": 10.98022174835205,
|
||||||
|
"learning_rate": 1.3856541105586545e-06,
|
||||||
|
"logits/chosen": -0.9942241856066296,
|
||||||
|
"logits/rejected": -0.9094278407875448,
|
||||||
|
"logps/chosen": -140.99659379323325,
|
||||||
|
"logps/rejected": -170.61128277248807,
|
||||||
|
"loss": 0.1790446400642395,
|
||||||
|
"mean_token_accuracy": 0.49307963914341396,
|
||||||
|
"num_tokens": 390815.0,
|
||||||
|
"rewards/accuracies": 0.9583333333333334,
|
||||||
|
"rewards/chosen": 0.6758623485349946,
|
||||||
|
"rewards/margins": 2.894124209880829,
|
||||||
|
"rewards/rejected": -2.218261867761612,
|
||||||
|
"step": 130
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"entropy": 2.0529139041900635,
|
||||||
|
"epoch": 2.144,
|
||||||
|
"grad_norm": 10.028863906860352,
|
||||||
|
"learning_rate": 1.1839195928066101e-06,
|
||||||
|
"logits/chosen": -1.0788186523834689,
|
||||||
|
"logits/rejected": -0.986635527035572,
|
||||||
|
"logps/chosen": -145.5455855369568,
|
||||||
|
"logps/rejected": -172.8044556617737,
|
||||||
|
"loss": 0.12198150157928467,
|
||||||
|
"mean_token_accuracy": 0.49735122323036196,
|
||||||
|
"num_tokens": 457256.0,
|
||||||
|
"rewards/accuracies": 0.975,
|
||||||
|
"rewards/chosen": 0.6768257994204759,
|
||||||
|
"rewards/margins": 3.0055406376719476,
|
||||||
|
"rewards/rejected": -2.3287148490548133,
|
||||||
|
"step": 135
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"entropy": 2.014680004119873,
|
||||||
|
"epoch": 2.224,
|
||||||
|
"grad_norm": 6.895854473114014,
|
||||||
|
"learning_rate": 9.934134090518593e-07,
|
||||||
|
"logits/chosen": -1.1362325910319455,
|
||||||
|
"logits/rejected": -1.002460196026479,
|
||||||
|
"logps/chosen": -141.88460431098937,
|
||||||
|
"logps/rejected": -159.63373432159423,
|
||||||
|
"loss": 0.14334821701049805,
|
||||||
|
"mean_token_accuracy": 0.48404414057731626,
|
||||||
|
"num_tokens": 519814.0,
|
||||||
|
"rewards/accuracies": 0.95,
|
||||||
|
"rewards/chosen": 0.5280852797441185,
|
||||||
|
"rewards/margins": 2.968889817595482,
|
||||||
|
"rewards/rejected": -2.4408045306801798,
|
||||||
|
"step": 140
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"entropy": 1.9911637425422668,
|
||||||
|
"epoch": 2.304,
|
||||||
|
"grad_norm": 10.900385856628418,
|
||||||
|
"learning_rate": 8.157608908836071e-07,
|
||||||
|
"logits/chosen": -1.240284110529506,
|
||||||
|
"logits/rejected": -1.2038341543285929,
|
||||||
|
"logps/chosen": -149.32998390197753,
|
||||||
|
"logps/rejected": -179.72271556854247,
|
||||||
|
"loss": 0.11892527341842651,
|
||||||
|
"mean_token_accuracy": 0.48943382054567336,
|
||||||
|
"num_tokens": 583559.0,
|
||||||
|
"rewards/accuracies": 0.975,
|
||||||
|
"rewards/chosen": 0.5228322677314281,
|
||||||
|
"rewards/margins": 3.2153540045022964,
|
||||||
|
"rewards/rejected": -2.6925217270851136,
|
||||||
|
"step": 145
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"entropy": 2.0109089225530625,
|
||||||
|
"epoch": 2.384,
|
||||||
|
"grad_norm": 7.562666893005371,
|
||||||
|
"learning_rate": 6.524777069483526e-07,
|
||||||
|
"logits/chosen": -1.362585639730851,
|
||||||
|
"logits/rejected": -1.3277683060910546,
|
||||||
|
"logps/chosen": -152.4582862854004,
|
||||||
|
"logps/rejected": -190.31861305236816,
|
||||||
|
"loss": 0.1071157455444336,
|
||||||
|
"mean_token_accuracy": 0.47375648468732834,
|
||||||
|
"num_tokens": 655280.0,
|
||||||
|
"rewards/accuracies": 0.95,
|
||||||
|
"rewards/chosen": 0.5572835557162762,
|
||||||
|
"rewards/margins": 3.580769953131676,
|
||||||
|
"rewards/rejected": -3.023486441373825,
|
||||||
|
"step": 150
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 2.384,
|
||||||
|
"eval_entropy": 1.9984020476341247,
|
||||||
|
"eval_logits/chosen": -1.3867654543550485,
|
||||||
|
"eval_logits/rejected": -1.3457294671252336,
|
||||||
|
"eval_logps/chosen": -172.3030450744629,
|
||||||
|
"eval_logps/rejected": -165.6812739868164,
|
||||||
|
"eval_loss": 0.7880498766899109,
|
||||||
|
"eval_mean_token_accuracy": 0.451040216088295,
|
||||||
|
"eval_num_tokens": 655280.0,
|
||||||
|
"eval_rewards/accuracies": 0.54,
|
||||||
|
"eval_rewards/chosen": -1.0855237049460411,
|
||||||
|
"eval_rewards/margins": 0.1670554026067257,
|
||||||
|
"eval_rewards/rejected": -1.25257910490036,
|
||||||
|
"eval_runtime": 203.3449,
|
||||||
|
"eval_samples_per_second": 4.918,
|
||||||
|
"eval_steps_per_second": 1.229,
|
||||||
|
"step": 150
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"entropy": 1.898311933875084,
|
||||||
|
"epoch": 2.464,
|
||||||
|
"grad_norm": 11.360427856445312,
|
||||||
|
"learning_rate": 5.049569317994013e-07,
|
||||||
|
"logits/chosen": -1.4353495696293097,
|
||||||
|
"logits/rejected": -1.3758243692230172,
|
||||||
|
"logps/chosen": -141.70478200912476,
|
||||||
|
"logps/rejected": -200.1671936035156,
|
||||||
|
"loss": 0.09740959405899048,
|
||||||
|
"mean_token_accuracy": 0.4936802975833416,
|
||||||
|
"num_tokens": 724110.0,
|
||||||
|
"rewards/accuracies": 0.96875,
|
||||||
|
"rewards/chosen": 0.4263562995940447,
|
||||||
|
"rewards/margins": 3.59362336397171,
|
||||||
|
"rewards/rejected": -3.167267033457756,
|
||||||
|
"step": 155
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"entropy": 1.939386433362961,
|
||||||
|
"epoch": 2.544,
|
||||||
|
"grad_norm": 8.742157936096191,
|
||||||
|
"learning_rate": 3.7445716067596506e-07,
|
||||||
|
"logits/chosen": -1.4550549474115178,
|
||||||
|
"logits/rejected": -1.3327657359030232,
|
||||||
|
"logps/chosen": -132.01672496795655,
|
||||||
|
"logps/rejected": -159.5728401184082,
|
||||||
|
"loss": 0.10442214012145996,
|
||||||
|
"mean_token_accuracy": 0.4859817087650299,
|
||||||
|
"num_tokens": 787960.0,
|
||||||
|
"rewards/accuracies": 0.98125,
|
||||||
|
"rewards/chosen": 0.42371090203523637,
|
||||||
|
"rewards/margins": 3.1901376724243162,
|
||||||
|
"rewards/rejected": -2.7664267748594282,
|
||||||
|
"step": 160
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"entropy": 1.9298288524150848,
|
||||||
|
"epoch": 2.624,
|
||||||
|
"grad_norm": 6.683835029602051,
|
||||||
|
"learning_rate": 2.620917716123444e-07,
|
||||||
|
"logits/chosen": -1.5218572998191655,
|
||||||
|
"logits/rejected": -1.5028220084321073,
|
||||||
|
"logps/chosen": -146.9026556968689,
|
||||||
|
"logps/rejected": -182.4629114151001,
|
||||||
|
"loss": 0.11852377653121948,
|
||||||
|
"mean_token_accuracy": 0.5008164420723915,
|
||||||
|
"num_tokens": 854381.0,
|
||||||
|
"rewards/accuracies": 0.975,
|
||||||
|
"rewards/chosen": 0.340854674577713,
|
||||||
|
"rewards/margins": 3.3918783754110335,
|
||||||
|
"rewards/rejected": -3.051023706793785,
|
||||||
|
"step": 165
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"entropy": 1.9614687770605088,
|
||||||
|
"epoch": 2.7039999999999997,
|
||||||
|
"grad_norm": 8.654135704040527,
|
||||||
|
"learning_rate": 1.6881942648911077e-07,
|
||||||
|
"logits/chosen": -1.483899302043358,
|
||||||
|
"logits/rejected": -1.3984594015444671,
|
||||||
|
"logps/chosen": -165.00297966003419,
|
||||||
|
"logps/rejected": -223.27304649353027,
|
||||||
|
"loss": 0.10529448986053466,
|
||||||
|
"mean_token_accuracy": 0.4777096025645733,
|
||||||
|
"num_tokens": 922218.0,
|
||||||
|
"rewards/accuracies": 0.9625,
|
||||||
|
"rewards/chosen": 0.1868226437829435,
|
||||||
|
"rewards/margins": 3.6259077370166777,
|
||||||
|
"rewards/rejected": -3.439085066318512,
|
||||||
|
"step": 170
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"entropy": 1.947239425778389,
|
||||||
|
"epoch": 2.784,
|
||||||
|
"grad_norm": 8.482812881469727,
|
||||||
|
"learning_rate": 9.54358920679524e-08,
|
||||||
|
"logits/chosen": -1.5538378778015716,
|
||||||
|
"logits/rejected": -1.4374772630805093,
|
||||||
|
"logps/chosen": -167.4766321182251,
|
||||||
|
"logps/rejected": -185.17562103271484,
|
||||||
|
"loss": 0.10844930410385131,
|
||||||
|
"mean_token_accuracy": 0.48532758429646494,
|
||||||
|
"num_tokens": 989301.0,
|
||||||
|
"rewards/accuracies": 0.98125,
|
||||||
|
"rewards/chosen": 0.5600280098617076,
|
||||||
|
"rewards/margins": 3.5826878786087035,
|
||||||
|
"rewards/rejected": -3.0226599007844923,
|
||||||
|
"step": 175
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"entropy": 1.9497412502765656,
|
||||||
|
"epoch": 2.864,
|
||||||
|
"grad_norm": 10.068187713623047,
|
||||||
|
"learning_rate": 4.256725079024554e-08,
|
||||||
|
"logits/chosen": -1.5250048523452793,
|
||||||
|
"logits/rejected": -1.4418669213263424,
|
||||||
|
"logps/chosen": -154.15428905487062,
|
||||||
|
"logps/rejected": -163.1813151359558,
|
||||||
|
"loss": 0.13129655122756959,
|
||||||
|
"mean_token_accuracy": 0.4707398295402527,
|
||||||
|
"num_tokens": 1057139.0,
|
||||||
|
"rewards/accuracies": 0.9625,
|
||||||
|
"rewards/chosen": 0.4407920534722507,
|
||||||
|
"rewards/margins": 3.0125378251075743,
|
||||||
|
"rewards/rejected": -2.571745750308037,
|
||||||
|
"step": 180
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"entropy": 1.9340084820985795,
|
||||||
|
"epoch": 2.944,
|
||||||
|
"grad_norm": 9.49740219116211,
|
||||||
|
"learning_rate": 1.0664559262413831e-08,
|
||||||
|
"logits/chosen": -1.512207703275473,
|
||||||
|
"logits/rejected": -1.4001704843220182,
|
||||||
|
"logps/chosen": -157.45920839309693,
|
||||||
|
"logps/rejected": -180.69428844451903,
|
||||||
|
"loss": 0.09854952096939087,
|
||||||
|
"mean_token_accuracy": 0.4828268110752106,
|
||||||
|
"num_tokens": 1120692.0,
|
||||||
|
"rewards/accuracies": 0.96875,
|
||||||
|
"rewards/chosen": 0.4773704446852207,
|
||||||
|
"rewards/margins": 3.379485607147217,
|
||||||
|
"rewards/rejected": -2.9021151691675184,
|
||||||
|
"step": 185
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 3.0,
|
||||||
|
"eval_entropy": 1.9640341634750367,
|
||||||
|
"eval_logits/chosen": -1.5042375842896538,
|
||||||
|
"eval_logits/rejected": -1.4667329044378787,
|
||||||
|
"eval_logps/chosen": -173.94541304016113,
|
||||||
|
"eval_logps/rejected": -167.3245940551758,
|
||||||
|
"eval_loss": 0.8192874193191528,
|
||||||
|
"eval_mean_token_accuracy": 0.44932303857803346,
|
||||||
|
"eval_num_tokens": 1169762.0,
|
||||||
|
"eval_rewards/accuracies": 0.534,
|
||||||
|
"eval_rewards/chosen": -1.2497604206800461,
|
||||||
|
"eval_rewards/margins": 0.16715064680576325,
|
||||||
|
"eval_rewards/rejected": -1.4169110633730888,
|
||||||
|
"eval_runtime": 203.5561,
|
||||||
|
"eval_samples_per_second": 4.913,
|
||||||
|
"eval_steps_per_second": 1.228,
|
||||||
|
"step": 189
|
||||||
|
}
|
||||||
|
]
|
||||||
3
model.safetensors
Normal file
3
model.safetensors
Normal file
@@ -0,0 +1,3 @@
|
|||||||
|
version https://git-lfs.github.com/spec/v1
|
||||||
|
oid sha256:fec90aa0de29c34e29db4101176e95b38fe7658c8e510641bbfaaa260772c186
|
||||||
|
size 1621370224
|
||||||
250557
tokenizer.json
Normal file
250557
tokenizer.json
Normal file
File diff suppressed because it is too large
Load Diff
14
tokenizer_config.json
Normal file
14
tokenizer_config.json
Normal file
@@ -0,0 +1,14 @@
|
|||||||
|
{
|
||||||
|
"add_prefix_space": false,
|
||||||
|
"backend": "tokenizers",
|
||||||
|
"bos_token": "<|endoftext|>",
|
||||||
|
"eos_token": "<|endoftext|>",
|
||||||
|
"errors": "replace",
|
||||||
|
"is_local": false,
|
||||||
|
"local_files_only": false,
|
||||||
|
"model_max_length": 1000000000000000019884624838656,
|
||||||
|
"pad_token": "<|padding|>",
|
||||||
|
"tokenizer_class": "GPTNeoXTokenizer",
|
||||||
|
"trim_offsets": true,
|
||||||
|
"unk_token": "<|endoftext|>"
|
||||||
|
}
|
||||||
3
training_args.bin
Normal file
3
training_args.bin
Normal file
@@ -0,0 +1,3 @@
|
|||||||
|
version https://git-lfs.github.com/spec/v1
|
||||||
|
oid sha256:e9c10bdc1b2e405fee996676bbed765046a6e06fe0fdaa6030fe444d7e535e6c
|
||||||
|
size 5841
|
||||||
Reference in New Issue
Block a user