From 3b4ee4fc4204427ed81201cf3dd187ab0ff0bdeb Mon Sep 17 00:00:00 2001 From: ModelHub XC Date: Mon, 1 Jun 2026 02:55:12 +0800 Subject: [PATCH] =?UTF-8?q?=E5=88=9D=E5=A7=8B=E5=8C=96=E9=A1=B9=E7=9B=AE?= =?UTF-8?q?=EF=BC=8C=E7=94=B1ModelHub=20XC=E7=A4=BE=E5=8C=BA=E6=8F=90?= =?UTF-8?q?=E4=BE=9B=E6=A8=A1=E5=9E=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Model: declare-lab/starling-7B Source: Original Platform --- .gitattributes | 37 ++ README.md | 175 ++++++ config.json | 23 + configuration.json | 1 + generation_config.json | 7 + pytorch_model-00001-of-00002.bin | 3 + pytorch_model-00002-of-00002.bin | 3 + pytorch_model.bin.index.json | 330 +++++++++++ special_tokens_map.json | 24 + tokenizer.model | 3 + tokenizer_config.json | 34 ++ trainer_state.json | 971 +++++++++++++++++++++++++++++++ 12 files changed, 1611 insertions(+) create mode 100644 .gitattributes create mode 100644 README.md create mode 100644 config.json create mode 100644 configuration.json create mode 100644 generation_config.json create mode 100644 pytorch_model-00001-of-00002.bin create mode 100644 pytorch_model-00002-of-00002.bin create mode 100644 pytorch_model.bin.index.json create mode 100644 special_tokens_map.json create mode 100644 tokenizer.model create mode 100644 tokenizer_config.json create mode 100644 trainer_state.json diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..e512815 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,37 @@ +*.7z filter=lfs diff=lfs merge=lfs -text +*.arrow filter=lfs diff=lfs merge=lfs -text +*.bin filter=lfs diff=lfs merge=lfs -text +*.bz2 filter=lfs diff=lfs merge=lfs -text +*.ckpt filter=lfs diff=lfs merge=lfs -text +*.ftz filter=lfs diff=lfs merge=lfs -text +*.gz filter=lfs diff=lfs merge=lfs -text +*.h5 filter=lfs diff=lfs merge=lfs -text +*.joblib filter=lfs diff=lfs merge=lfs -text +*.lfs.* filter=lfs diff=lfs merge=lfs -text +*.mlmodel filter=lfs diff=lfs merge=lfs -text +*.model filter=lfs diff=lfs merge=lfs -text +*.msgpack filter=lfs diff=lfs merge=lfs -text +*.npy filter=lfs diff=lfs merge=lfs -text +*.npz filter=lfs diff=lfs merge=lfs -text +*.onnx filter=lfs diff=lfs merge=lfs -text +*.ot filter=lfs diff=lfs merge=lfs -text +*.parquet filter=lfs diff=lfs merge=lfs -text +*.pb filter=lfs diff=lfs merge=lfs -text +*.pickle filter=lfs diff=lfs merge=lfs -text +*.pkl filter=lfs diff=lfs merge=lfs -text +*.pt filter=lfs diff=lfs merge=lfs -text +*.pth filter=lfs diff=lfs merge=lfs -text +*.rar filter=lfs diff=lfs merge=lfs -text +*.safetensors filter=lfs diff=lfs merge=lfs -text +saved_model/**/* filter=lfs diff=lfs merge=lfs -text +*.tar.* filter=lfs diff=lfs merge=lfs -text +*.tar filter=lfs diff=lfs merge=lfs -text +*.tflite filter=lfs diff=lfs merge=lfs -text +*.tgz filter=lfs diff=lfs merge=lfs -text +*.wasm filter=lfs diff=lfs merge=lfs -text +*.xz filter=lfs diff=lfs merge=lfs -text +*.zip filter=lfs diff=lfs merge=lfs -text +*.zst filter=lfs diff=lfs merge=lfs -text +*tfevents* filter=lfs diff=lfs merge=lfs -text +pytorch_model-00001-of-00002.bin filter=lfs diff=lfs merge=lfs -text +pytorch_model-00002-of-00002.bin filter=lfs diff=lfs merge=lfs -text diff --git a/README.md b/README.md new file mode 100644 index 0000000..6b2a7c7 --- /dev/null +++ b/README.md @@ -0,0 +1,175 @@ +--- +license: apache-2.0 +datasets: +- anon8231489123/ShareGPT_Vicuna_unfiltered +- declare-lab/HarmfulQA +model-index: +- name: starling-7B + results: + - task: + type: text-generation + name: Text Generation + dataset: + name: AI2 Reasoning Challenge (25-Shot) + type: ai2_arc + config: ARC-Challenge + split: test + args: + num_few_shot: 25 + metrics: + - type: acc_norm + value: 51.02 + name: normalized accuracy + source: + url: https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard?query=declare-lab/starling-7B + name: Open LLM Leaderboard + - task: + type: text-generation + name: Text Generation + dataset: + name: HellaSwag (10-Shot) + type: hellaswag + split: validation + args: + num_few_shot: 10 + metrics: + - type: acc_norm + value: 76.77 + name: normalized accuracy + source: + url: https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard?query=declare-lab/starling-7B + name: Open LLM Leaderboard + - task: + type: text-generation + name: Text Generation + dataset: + name: MMLU (5-Shot) + type: cais/mmlu + config: all + split: test + args: + num_few_shot: 5 + metrics: + - type: acc + value: 47.75 + name: accuracy + source: + url: https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard?query=declare-lab/starling-7B + name: Open LLM Leaderboard + - task: + type: text-generation + name: Text Generation + dataset: + name: TruthfulQA (0-shot) + type: truthful_qa + config: multiple_choice + split: validation + args: + num_few_shot: 0 + metrics: + - type: mc2 + value: 48.18 + source: + url: https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard?query=declare-lab/starling-7B + name: Open LLM Leaderboard + - task: + type: text-generation + name: Text Generation + dataset: + name: Winogrande (5-shot) + type: winogrande + config: winogrande_xl + split: validation + args: + num_few_shot: 5 + metrics: + - type: acc + value: 70.56 + name: accuracy + source: + url: https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard?query=declare-lab/starling-7B + name: Open LLM Leaderboard + - task: + type: text-generation + name: Text Generation + dataset: + name: GSM8k (5-shot) + type: gsm8k + config: main + split: test + args: + num_few_shot: 5 + metrics: + - type: acc + value: 10.08 + name: accuracy + source: + url: https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard?query=declare-lab/starling-7B + name: Open LLM Leaderboard +--- +[**Paper**](https://arxiv.org/abs/2308.09662) | [**Github**](https://github.com/declare-lab/red-instruct) | [**Dataset**](https://huggingface.co/datasets/declare-lab/HarmfulQA)| [**Model**](https://huggingface.co/declare-lab/starling-7B) + + +> 📣 Update 2/02/24: Introducing Resta: **Safety Re-alignment of Language Models**. [**Paper**](https://arxiv.org/abs/2402.11746) [**Github**](https://github.com/declare-lab/resta) [**Dataset**](https://huggingface.co/datasets/declare-lab/CategoricalHarmfulQ) + +As a part of our research efforts to make LLMs safer, we created **Starling**. It is obtained by fine-tuning Vicuna-7B on [**HarmfulQA**](https://huggingface.co/datasets/declare-lab/HarmfulQA), a ChatGPT-distilled dataset that we collected using the Chain of Utterances (CoU) prompt. More details are in our paper [**Red-Teaming Large Language Models using Chain of Utterances for Safety-Alignment**](https://arxiv.org/abs/2308.09662) + +Image + +Experimental results on several safety benchmark datasets indicate that **Starling** is a safer model compared to the baseline model, Vicuna. + +Image + +

Experimental Results

+ +Compared to Vicuna, **Avg. 5.2% reduction in Attack Success Rate** (ASR) on DangerousQA and HarmfulQA using three different prompts.** + +Compared to Vicuna, **Avg. 3-7% improvement in HHH score** measured on BBH-HHH benchmark.** + +Image + +TruthfulQA (MC2): **48.90 vs Vicuna's 47.00** + +MMLU (5-shot): **46.69 vs Vicuna's 47.18** + +BBH (3-shot): **33.47 vs Vicuna's 33.05** + +

Jailbreak Prompt for harmfulness eval using Red Eval as reported in the paper

+ +This jailbreak prompt (termed as Chain of Utterances (CoU) prompt in the paper) shows a 65% Attack Success Rate (ASR) on GPT-4 and 72% on ChatGPT. + +Image + +

HarmfulQA Data Collection

+ +We also release our **HarmfulQA** dataset with 1,960 harmful questions (converting 10 topics-10 subtopics) for red-teaming as well as conversations based on them used in model safety alignment, more details [**here**](https://huggingface.co/datasets/declare-lab/HarmfulQA). The following figure describes the data collection process. + +Image + +_Note: This model is referred to as Starling (Blue) in the paper. We shall soon release Starling (Blue-Red) which was trained on harmful data using an objective function that helps the model learn from the red (harmful) response data._ + +## Citation + +```bibtex +@misc{bhardwaj2023redteaming, + title={Red-Teaming Large Language Models using Chain of Utterances for Safety-Alignment}, + author={Rishabh Bhardwaj and Soujanya Poria}, + year={2023}, + eprint={2308.09662}, + archivePrefix={arXiv}, + primaryClass={cs.CL} +} +``` +# [Open LLM Leaderboard Evaluation Results](https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard) +Detailed results can be found [here](https://huggingface.co/datasets/open-llm-leaderboard/details_declare-lab__starling-7B) + +| Metric |Value| +|---------------------------------|----:| +|Avg. |50.73| +|AI2 Reasoning Challenge (25-Shot)|51.02| +|HellaSwag (10-Shot) |76.77| +|MMLU (5-Shot) |47.75| +|TruthfulQA (0-shot) |48.18| +|Winogrande (5-shot) |70.56| +|GSM8k (5-shot) |10.08| + diff --git a/config.json b/config.json new file mode 100644 index 0000000..5ee4adc --- /dev/null +++ b/config.json @@ -0,0 +1,23 @@ +{ + "_name_or_path": "lmsys/vicuna-7b-v1.3", + "architectures": [ + "LlamaForCausalLM" + ], + "bos_token_id": 1, + "eos_token_id": 2, + "hidden_act": "silu", + "hidden_size": 4096, + "initializer_range": 0.02, + "intermediate_size": 11008, + "max_position_embeddings": 2048, + "model_type": "llama", + "num_attention_heads": 32, + "num_hidden_layers": 32, + "pad_token_id": 0, + "rms_norm_eps": 1e-06, + "tie_word_embeddings": false, + "torch_dtype": "float16", + "transformers_version": "4.29.0", + "use_cache": false, + "vocab_size": 32000 +} diff --git a/configuration.json b/configuration.json new file mode 100644 index 0000000..bbeeda1 --- /dev/null +++ b/configuration.json @@ -0,0 +1 @@ +{"framework": "pytorch", "task": "text-generation", "allow_remote": true} \ No newline at end of file diff --git a/generation_config.json b/generation_config.json new file mode 100644 index 0000000..66f8de0 --- /dev/null +++ b/generation_config.json @@ -0,0 +1,7 @@ +{ + "_from_model_config": true, + "bos_token_id": 1, + "eos_token_id": 2, + "pad_token_id": 0, + "transformers_version": "4.29.0" +} diff --git a/pytorch_model-00001-of-00002.bin b/pytorch_model-00001-of-00002.bin new file mode 100644 index 0000000..832e527 --- /dev/null +++ b/pytorch_model-00001-of-00002.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:befd55f89dcec43643449284986c6c44c3d4c3093b12f6464d813d55062daafd +size 13476873049 diff --git a/pytorch_model-00002-of-00002.bin b/pytorch_model-00002-of-00002.bin new file mode 100644 index 0000000..30b27a3 --- /dev/null +++ b/pytorch_model-00002-of-00002.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4e7998c25896d2c7ad6b0f8d4572809e25733224bca73adfe557fdc0f346d0ef +size 13476845562 diff --git a/pytorch_model.bin.index.json b/pytorch_model.bin.index.json new file mode 100644 index 0000000..d7f2d6c --- /dev/null +++ b/pytorch_model.bin.index.json @@ -0,0 +1,330 @@ +{ + "metadata": { + "total_size": 13476835328 + }, + "weight_map": { + "lm_head.weight": "pytorch_model-00002-of-00002.bin", + "model.embed_tokens.weight": "pytorch_model-00001-of-00002.bin", + "model.layers.0.input_layernorm.weight": "pytorch_model-00001-of-00002.bin", + "model.layers.0.mlp.down_proj.weight": "pytorch_model-00001-of-00002.bin", + "model.layers.0.mlp.gate_proj.weight": "pytorch_model-00001-of-00002.bin", + "model.layers.0.mlp.up_proj.weight": "pytorch_model-00001-of-00002.bin", + "model.layers.0.post_attention_layernorm.weight": "pytorch_model-00001-of-00002.bin", + "model.layers.0.self_attn.k_proj.weight": "pytorch_model-00001-of-00002.bin", + "model.layers.0.self_attn.o_proj.weight": "pytorch_model-00001-of-00002.bin", + "model.layers.0.self_attn.q_proj.weight": "pytorch_model-00001-of-00002.bin", + "model.layers.0.self_attn.rotary_emb.inv_freq": "pytorch_model-00001-of-00002.bin", + "model.layers.0.self_attn.v_proj.weight": "pytorch_model-00001-of-00002.bin", + "model.layers.1.input_layernorm.weight": "pytorch_model-00001-of-00002.bin", + "model.layers.1.mlp.down_proj.weight": "pytorch_model-00001-of-00002.bin", + "model.layers.1.mlp.gate_proj.weight": "pytorch_model-00001-of-00002.bin", + "model.layers.1.mlp.up_proj.weight": "pytorch_model-00001-of-00002.bin", + "model.layers.1.post_attention_layernorm.weight": "pytorch_model-00001-of-00002.bin", + "model.layers.1.self_attn.k_proj.weight": "pytorch_model-00001-of-00002.bin", + "model.layers.1.self_attn.o_proj.weight": "pytorch_model-00001-of-00002.bin", + "model.layers.1.self_attn.q_proj.weight": "pytorch_model-00001-of-00002.bin", + "model.layers.1.self_attn.rotary_emb.inv_freq": "pytorch_model-00001-of-00002.bin", + "model.layers.1.self_attn.v_proj.weight": "pytorch_model-00001-of-00002.bin", + "model.layers.10.input_layernorm.weight": "pytorch_model-00001-of-00002.bin", + "model.layers.10.mlp.down_proj.weight": "pytorch_model-00001-of-00002.bin", + "model.layers.10.mlp.gate_proj.weight": "pytorch_model-00001-of-00002.bin", + "model.layers.10.mlp.up_proj.weight": "pytorch_model-00001-of-00002.bin", + "model.layers.10.post_attention_layernorm.weight": "pytorch_model-00001-of-00002.bin", + "model.layers.10.self_attn.k_proj.weight": "pytorch_model-00001-of-00002.bin", + "model.layers.10.self_attn.o_proj.weight": "pytorch_model-00001-of-00002.bin", + "model.layers.10.self_attn.q_proj.weight": "pytorch_model-00001-of-00002.bin", + "model.layers.10.self_attn.rotary_emb.inv_freq": "pytorch_model-00001-of-00002.bin", + "model.layers.10.self_attn.v_proj.weight": "pytorch_model-00001-of-00002.bin", + "model.layers.11.input_layernorm.weight": "pytorch_model-00001-of-00002.bin", + "model.layers.11.mlp.down_proj.weight": "pytorch_model-00001-of-00002.bin", + "model.layers.11.mlp.gate_proj.weight": "pytorch_model-00001-of-00002.bin", + "model.layers.11.mlp.up_proj.weight": "pytorch_model-00001-of-00002.bin", + "model.layers.11.post_attention_layernorm.weight": "pytorch_model-00001-of-00002.bin", + "model.layers.11.self_attn.k_proj.weight": "pytorch_model-00001-of-00002.bin", + "model.layers.11.self_attn.o_proj.weight": "pytorch_model-00001-of-00002.bin", + "model.layers.11.self_attn.q_proj.weight": "pytorch_model-00001-of-00002.bin", + "model.layers.11.self_attn.rotary_emb.inv_freq": "pytorch_model-00001-of-00002.bin", + "model.layers.11.self_attn.v_proj.weight": "pytorch_model-00001-of-00002.bin", + "model.layers.12.input_layernorm.weight": "pytorch_model-00001-of-00002.bin", + "model.layers.12.mlp.down_proj.weight": "pytorch_model-00001-of-00002.bin", + "model.layers.12.mlp.gate_proj.weight": "pytorch_model-00001-of-00002.bin", + "model.layers.12.mlp.up_proj.weight": "pytorch_model-00001-of-00002.bin", + "model.layers.12.post_attention_layernorm.weight": "pytorch_model-00001-of-00002.bin", + "model.layers.12.self_attn.k_proj.weight": "pytorch_model-00001-of-00002.bin", + "model.layers.12.self_attn.o_proj.weight": "pytorch_model-00001-of-00002.bin", + "model.layers.12.self_attn.q_proj.weight": "pytorch_model-00001-of-00002.bin", + "model.layers.12.self_attn.rotary_emb.inv_freq": "pytorch_model-00001-of-00002.bin", + "model.layers.12.self_attn.v_proj.weight": "pytorch_model-00001-of-00002.bin", + "model.layers.13.input_layernorm.weight": "pytorch_model-00001-of-00002.bin", + "model.layers.13.mlp.down_proj.weight": "pytorch_model-00001-of-00002.bin", + "model.layers.13.mlp.gate_proj.weight": "pytorch_model-00001-of-00002.bin", + "model.layers.13.mlp.up_proj.weight": "pytorch_model-00001-of-00002.bin", + "model.layers.13.post_attention_layernorm.weight": "pytorch_model-00001-of-00002.bin", + "model.layers.13.self_attn.k_proj.weight": "pytorch_model-00001-of-00002.bin", + "model.layers.13.self_attn.o_proj.weight": "pytorch_model-00001-of-00002.bin", + "model.layers.13.self_attn.q_proj.weight": "pytorch_model-00001-of-00002.bin", + "model.layers.13.self_attn.rotary_emb.inv_freq": "pytorch_model-00001-of-00002.bin", + "model.layers.13.self_attn.v_proj.weight": "pytorch_model-00001-of-00002.bin", + "model.layers.14.input_layernorm.weight": "pytorch_model-00001-of-00002.bin", + "model.layers.14.mlp.down_proj.weight": "pytorch_model-00001-of-00002.bin", + "model.layers.14.mlp.gate_proj.weight": "pytorch_model-00001-of-00002.bin", + "model.layers.14.mlp.up_proj.weight": "pytorch_model-00001-of-00002.bin", + "model.layers.14.post_attention_layernorm.weight": "pytorch_model-00001-of-00002.bin", + "model.layers.14.self_attn.k_proj.weight": "pytorch_model-00001-of-00002.bin", + "model.layers.14.self_attn.o_proj.weight": "pytorch_model-00001-of-00002.bin", + "model.layers.14.self_attn.q_proj.weight": "pytorch_model-00001-of-00002.bin", + "model.layers.14.self_attn.rotary_emb.inv_freq": "pytorch_model-00001-of-00002.bin", + "model.layers.14.self_attn.v_proj.weight": "pytorch_model-00001-of-00002.bin", + "model.layers.15.input_layernorm.weight": "pytorch_model-00001-of-00002.bin", + "model.layers.15.mlp.down_proj.weight": "pytorch_model-00001-of-00002.bin", + "model.layers.15.mlp.gate_proj.weight": "pytorch_model-00001-of-00002.bin", + "model.layers.15.mlp.up_proj.weight": "pytorch_model-00001-of-00002.bin", + "model.layers.15.post_attention_layernorm.weight": "pytorch_model-00001-of-00002.bin", + "model.layers.15.self_attn.k_proj.weight": "pytorch_model-00001-of-00002.bin", + "model.layers.15.self_attn.o_proj.weight": "pytorch_model-00001-of-00002.bin", + "model.layers.15.self_attn.q_proj.weight": "pytorch_model-00001-of-00002.bin", + "model.layers.15.self_attn.rotary_emb.inv_freq": "pytorch_model-00001-of-00002.bin", + "model.layers.15.self_attn.v_proj.weight": "pytorch_model-00001-of-00002.bin", + "model.layers.16.input_layernorm.weight": "pytorch_model-00001-of-00002.bin", + "model.layers.16.mlp.down_proj.weight": "pytorch_model-00001-of-00002.bin", + "model.layers.16.mlp.gate_proj.weight": "pytorch_model-00001-of-00002.bin", + "model.layers.16.mlp.up_proj.weight": "pytorch_model-00001-of-00002.bin", + "model.layers.16.post_attention_layernorm.weight": "pytorch_model-00001-of-00002.bin", + "model.layers.16.self_attn.k_proj.weight": "pytorch_model-00001-of-00002.bin", + "model.layers.16.self_attn.o_proj.weight": "pytorch_model-00001-of-00002.bin", + "model.layers.16.self_attn.q_proj.weight": "pytorch_model-00001-of-00002.bin", + "model.layers.16.self_attn.rotary_emb.inv_freq": "pytorch_model-00001-of-00002.bin", + "model.layers.16.self_attn.v_proj.weight": "pytorch_model-00001-of-00002.bin", + "model.layers.17.input_layernorm.weight": "pytorch_model-00001-of-00002.bin", + "model.layers.17.mlp.down_proj.weight": "pytorch_model-00001-of-00002.bin", + "model.layers.17.mlp.gate_proj.weight": "pytorch_model-00001-of-00002.bin", + "model.layers.17.mlp.up_proj.weight": "pytorch_model-00001-of-00002.bin", + "model.layers.17.post_attention_layernorm.weight": "pytorch_model-00001-of-00002.bin", + "model.layers.17.self_attn.k_proj.weight": "pytorch_model-00001-of-00002.bin", + "model.layers.17.self_attn.o_proj.weight": "pytorch_model-00001-of-00002.bin", + "model.layers.17.self_attn.q_proj.weight": "pytorch_model-00001-of-00002.bin", + "model.layers.17.self_attn.rotary_emb.inv_freq": "pytorch_model-00001-of-00002.bin", + "model.layers.17.self_attn.v_proj.weight": "pytorch_model-00001-of-00002.bin", + "model.layers.18.input_layernorm.weight": "pytorch_model-00001-of-00002.bin", + "model.layers.18.mlp.down_proj.weight": "pytorch_model-00001-of-00002.bin", + "model.layers.18.mlp.gate_proj.weight": "pytorch_model-00001-of-00002.bin", + "model.layers.18.mlp.up_proj.weight": "pytorch_model-00001-of-00002.bin", + "model.layers.18.post_attention_layernorm.weight": "pytorch_model-00001-of-00002.bin", + "model.layers.18.self_attn.k_proj.weight": "pytorch_model-00001-of-00002.bin", + "model.layers.18.self_attn.o_proj.weight": "pytorch_model-00001-of-00002.bin", + "model.layers.18.self_attn.q_proj.weight": "pytorch_model-00001-of-00002.bin", + "model.layers.18.self_attn.rotary_emb.inv_freq": "pytorch_model-00001-of-00002.bin", + "model.layers.18.self_attn.v_proj.weight": "pytorch_model-00001-of-00002.bin", + "model.layers.19.input_layernorm.weight": "pytorch_model-00001-of-00002.bin", + "model.layers.19.mlp.down_proj.weight": "pytorch_model-00001-of-00002.bin", + "model.layers.19.mlp.gate_proj.weight": "pytorch_model-00001-of-00002.bin", + "model.layers.19.mlp.up_proj.weight": "pytorch_model-00001-of-00002.bin", + "model.layers.19.post_attention_layernorm.weight": "pytorch_model-00001-of-00002.bin", + "model.layers.19.self_attn.k_proj.weight": "pytorch_model-00001-of-00002.bin", + "model.layers.19.self_attn.o_proj.weight": "pytorch_model-00001-of-00002.bin", + "model.layers.19.self_attn.q_proj.weight": "pytorch_model-00001-of-00002.bin", + "model.layers.19.self_attn.rotary_emb.inv_freq": "pytorch_model-00001-of-00002.bin", + "model.layers.19.self_attn.v_proj.weight": "pytorch_model-00001-of-00002.bin", + "model.layers.2.input_layernorm.weight": "pytorch_model-00001-of-00002.bin", + "model.layers.2.mlp.down_proj.weight": "pytorch_model-00001-of-00002.bin", + "model.layers.2.mlp.gate_proj.weight": "pytorch_model-00001-of-00002.bin", + "model.layers.2.mlp.up_proj.weight": "pytorch_model-00001-of-00002.bin", + "model.layers.2.post_attention_layernorm.weight": "pytorch_model-00001-of-00002.bin", + "model.layers.2.self_attn.k_proj.weight": "pytorch_model-00001-of-00002.bin", + "model.layers.2.self_attn.o_proj.weight": "pytorch_model-00001-of-00002.bin", + "model.layers.2.self_attn.q_proj.weight": "pytorch_model-00001-of-00002.bin", + "model.layers.2.self_attn.rotary_emb.inv_freq": "pytorch_model-00001-of-00002.bin", + "model.layers.2.self_attn.v_proj.weight": "pytorch_model-00001-of-00002.bin", + "model.layers.20.input_layernorm.weight": "pytorch_model-00001-of-00002.bin", + "model.layers.20.mlp.down_proj.weight": "pytorch_model-00001-of-00002.bin", + "model.layers.20.mlp.gate_proj.weight": "pytorch_model-00001-of-00002.bin", + "model.layers.20.mlp.up_proj.weight": "pytorch_model-00001-of-00002.bin", + "model.layers.20.post_attention_layernorm.weight": "pytorch_model-00001-of-00002.bin", + "model.layers.20.self_attn.k_proj.weight": "pytorch_model-00001-of-00002.bin", + "model.layers.20.self_attn.o_proj.weight": "pytorch_model-00001-of-00002.bin", + "model.layers.20.self_attn.q_proj.weight": "pytorch_model-00001-of-00002.bin", + "model.layers.20.self_attn.rotary_emb.inv_freq": "pytorch_model-00001-of-00002.bin", + "model.layers.20.self_attn.v_proj.weight": "pytorch_model-00001-of-00002.bin", + "model.layers.21.input_layernorm.weight": "pytorch_model-00001-of-00002.bin", + "model.layers.21.mlp.down_proj.weight": "pytorch_model-00001-of-00002.bin", + "model.layers.21.mlp.gate_proj.weight": "pytorch_model-00001-of-00002.bin", + "model.layers.21.mlp.up_proj.weight": "pytorch_model-00001-of-00002.bin", + "model.layers.21.post_attention_layernorm.weight": "pytorch_model-00001-of-00002.bin", + "model.layers.21.self_attn.k_proj.weight": "pytorch_model-00001-of-00002.bin", + "model.layers.21.self_attn.o_proj.weight": "pytorch_model-00001-of-00002.bin", + "model.layers.21.self_attn.q_proj.weight": "pytorch_model-00001-of-00002.bin", + "model.layers.21.self_attn.rotary_emb.inv_freq": "pytorch_model-00001-of-00002.bin", + "model.layers.21.self_attn.v_proj.weight": "pytorch_model-00001-of-00002.bin", + "model.layers.22.input_layernorm.weight": "pytorch_model-00001-of-00002.bin", + "model.layers.22.mlp.down_proj.weight": "pytorch_model-00001-of-00002.bin", + "model.layers.22.mlp.gate_proj.weight": "pytorch_model-00001-of-00002.bin", + "model.layers.22.mlp.up_proj.weight": "pytorch_model-00001-of-00002.bin", + "model.layers.22.post_attention_layernorm.weight": "pytorch_model-00001-of-00002.bin", + "model.layers.22.self_attn.k_proj.weight": "pytorch_model-00001-of-00002.bin", + "model.layers.22.self_attn.o_proj.weight": "pytorch_model-00001-of-00002.bin", + "model.layers.22.self_attn.q_proj.weight": "pytorch_model-00001-of-00002.bin", + "model.layers.22.self_attn.rotary_emb.inv_freq": "pytorch_model-00001-of-00002.bin", + "model.layers.22.self_attn.v_proj.weight": "pytorch_model-00001-of-00002.bin", + "model.layers.23.input_layernorm.weight": "pytorch_model-00001-of-00002.bin", + "model.layers.23.mlp.down_proj.weight": "pytorch_model-00001-of-00002.bin", + "model.layers.23.mlp.gate_proj.weight": "pytorch_model-00001-of-00002.bin", + "model.layers.23.mlp.up_proj.weight": "pytorch_model-00001-of-00002.bin", + "model.layers.23.post_attention_layernorm.weight": "pytorch_model-00001-of-00002.bin", + "model.layers.23.self_attn.k_proj.weight": "pytorch_model-00001-of-00002.bin", + "model.layers.23.self_attn.o_proj.weight": "pytorch_model-00001-of-00002.bin", + "model.layers.23.self_attn.q_proj.weight": "pytorch_model-00001-of-00002.bin", + "model.layers.23.self_attn.rotary_emb.inv_freq": "pytorch_model-00001-of-00002.bin", + "model.layers.23.self_attn.v_proj.weight": "pytorch_model-00001-of-00002.bin", + "model.layers.24.input_layernorm.weight": "pytorch_model-00002-of-00002.bin", + "model.layers.24.mlp.down_proj.weight": "pytorch_model-00002-of-00002.bin", + "model.layers.24.mlp.gate_proj.weight": "pytorch_model-00002-of-00002.bin", + "model.layers.24.mlp.up_proj.weight": "pytorch_model-00002-of-00002.bin", + "model.layers.24.post_attention_layernorm.weight": "pytorch_model-00002-of-00002.bin", + "model.layers.24.self_attn.k_proj.weight": "pytorch_model-00002-of-00002.bin", + "model.layers.24.self_attn.o_proj.weight": "pytorch_model-00002-of-00002.bin", + "model.layers.24.self_attn.q_proj.weight": "pytorch_model-00002-of-00002.bin", + "model.layers.24.self_attn.rotary_emb.inv_freq": "pytorch_model-00002-of-00002.bin", + "model.layers.24.self_attn.v_proj.weight": "pytorch_model-00002-of-00002.bin", + "model.layers.25.input_layernorm.weight": "pytorch_model-00002-of-00002.bin", + "model.layers.25.mlp.down_proj.weight": "pytorch_model-00002-of-00002.bin", + "model.layers.25.mlp.gate_proj.weight": "pytorch_model-00002-of-00002.bin", + "model.layers.25.mlp.up_proj.weight": "pytorch_model-00002-of-00002.bin", + "model.layers.25.post_attention_layernorm.weight": "pytorch_model-00002-of-00002.bin", + "model.layers.25.self_attn.k_proj.weight": "pytorch_model-00002-of-00002.bin", + "model.layers.25.self_attn.o_proj.weight": "pytorch_model-00002-of-00002.bin", + "model.layers.25.self_attn.q_proj.weight": "pytorch_model-00002-of-00002.bin", + "model.layers.25.self_attn.rotary_emb.inv_freq": "pytorch_model-00002-of-00002.bin", + "model.layers.25.self_attn.v_proj.weight": "pytorch_model-00002-of-00002.bin", + "model.layers.26.input_layernorm.weight": "pytorch_model-00002-of-00002.bin", + "model.layers.26.mlp.down_proj.weight": "pytorch_model-00002-of-00002.bin", + "model.layers.26.mlp.gate_proj.weight": "pytorch_model-00002-of-00002.bin", + "model.layers.26.mlp.up_proj.weight": "pytorch_model-00002-of-00002.bin", + "model.layers.26.post_attention_layernorm.weight": "pytorch_model-00002-of-00002.bin", + "model.layers.26.self_attn.k_proj.weight": "pytorch_model-00002-of-00002.bin", + "model.layers.26.self_attn.o_proj.weight": "pytorch_model-00002-of-00002.bin", + "model.layers.26.self_attn.q_proj.weight": "pytorch_model-00002-of-00002.bin", + "model.layers.26.self_attn.rotary_emb.inv_freq": "pytorch_model-00002-of-00002.bin", + "model.layers.26.self_attn.v_proj.weight": "pytorch_model-00002-of-00002.bin", + "model.layers.27.input_layernorm.weight": "pytorch_model-00002-of-00002.bin", + "model.layers.27.mlp.down_proj.weight": "pytorch_model-00002-of-00002.bin", + "model.layers.27.mlp.gate_proj.weight": "pytorch_model-00002-of-00002.bin", + "model.layers.27.mlp.up_proj.weight": "pytorch_model-00002-of-00002.bin", + "model.layers.27.post_attention_layernorm.weight": "pytorch_model-00002-of-00002.bin", + "model.layers.27.self_attn.k_proj.weight": "pytorch_model-00002-of-00002.bin", + "model.layers.27.self_attn.o_proj.weight": "pytorch_model-00002-of-00002.bin", + "model.layers.27.self_attn.q_proj.weight": "pytorch_model-00002-of-00002.bin", + "model.layers.27.self_attn.rotary_emb.inv_freq": "pytorch_model-00002-of-00002.bin", + "model.layers.27.self_attn.v_proj.weight": "pytorch_model-00002-of-00002.bin", + "model.layers.28.input_layernorm.weight": "pytorch_model-00002-of-00002.bin", + "model.layers.28.mlp.down_proj.weight": "pytorch_model-00002-of-00002.bin", + "model.layers.28.mlp.gate_proj.weight": "pytorch_model-00002-of-00002.bin", + "model.layers.28.mlp.up_proj.weight": "pytorch_model-00002-of-00002.bin", + "model.layers.28.post_attention_layernorm.weight": "pytorch_model-00002-of-00002.bin", + "model.layers.28.self_attn.k_proj.weight": "pytorch_model-00002-of-00002.bin", + "model.layers.28.self_attn.o_proj.weight": "pytorch_model-00002-of-00002.bin", + "model.layers.28.self_attn.q_proj.weight": "pytorch_model-00002-of-00002.bin", + "model.layers.28.self_attn.rotary_emb.inv_freq": "pytorch_model-00002-of-00002.bin", + "model.layers.28.self_attn.v_proj.weight": "pytorch_model-00002-of-00002.bin", + "model.layers.29.input_layernorm.weight": "pytorch_model-00002-of-00002.bin", + "model.layers.29.mlp.down_proj.weight": "pytorch_model-00002-of-00002.bin", + "model.layers.29.mlp.gate_proj.weight": "pytorch_model-00002-of-00002.bin", + "model.layers.29.mlp.up_proj.weight": "pytorch_model-00002-of-00002.bin", + "model.layers.29.post_attention_layernorm.weight": "pytorch_model-00002-of-00002.bin", + "model.layers.29.self_attn.k_proj.weight": "pytorch_model-00002-of-00002.bin", + "model.layers.29.self_attn.o_proj.weight": "pytorch_model-00002-of-00002.bin", + "model.layers.29.self_attn.q_proj.weight": "pytorch_model-00002-of-00002.bin", + "model.layers.29.self_attn.rotary_emb.inv_freq": "pytorch_model-00002-of-00002.bin", + "model.layers.29.self_attn.v_proj.weight": "pytorch_model-00002-of-00002.bin", + "model.layers.3.input_layernorm.weight": "pytorch_model-00001-of-00002.bin", + "model.layers.3.mlp.down_proj.weight": "pytorch_model-00001-of-00002.bin", + "model.layers.3.mlp.gate_proj.weight": "pytorch_model-00001-of-00002.bin", + "model.layers.3.mlp.up_proj.weight": "pytorch_model-00001-of-00002.bin", + "model.layers.3.post_attention_layernorm.weight": "pytorch_model-00001-of-00002.bin", + "model.layers.3.self_attn.k_proj.weight": "pytorch_model-00001-of-00002.bin", + "model.layers.3.self_attn.o_proj.weight": "pytorch_model-00001-of-00002.bin", + "model.layers.3.self_attn.q_proj.weight": "pytorch_model-00001-of-00002.bin", + "model.layers.3.self_attn.rotary_emb.inv_freq": "pytorch_model-00001-of-00002.bin", + "model.layers.3.self_attn.v_proj.weight": "pytorch_model-00001-of-00002.bin", + "model.layers.30.input_layernorm.weight": "pytorch_model-00002-of-00002.bin", + "model.layers.30.mlp.down_proj.weight": "pytorch_model-00002-of-00002.bin", + "model.layers.30.mlp.gate_proj.weight": "pytorch_model-00002-of-00002.bin", + "model.layers.30.mlp.up_proj.weight": "pytorch_model-00002-of-00002.bin", + "model.layers.30.post_attention_layernorm.weight": "pytorch_model-00002-of-00002.bin", + "model.layers.30.self_attn.k_proj.weight": "pytorch_model-00002-of-00002.bin", + "model.layers.30.self_attn.o_proj.weight": "pytorch_model-00002-of-00002.bin", + "model.layers.30.self_attn.q_proj.weight": "pytorch_model-00002-of-00002.bin", + "model.layers.30.self_attn.rotary_emb.inv_freq": "pytorch_model-00002-of-00002.bin", + "model.layers.30.self_attn.v_proj.weight": "pytorch_model-00002-of-00002.bin", + "model.layers.31.input_layernorm.weight": "pytorch_model-00002-of-00002.bin", + "model.layers.31.mlp.down_proj.weight": "pytorch_model-00002-of-00002.bin", + "model.layers.31.mlp.gate_proj.weight": "pytorch_model-00002-of-00002.bin", + "model.layers.31.mlp.up_proj.weight": "pytorch_model-00002-of-00002.bin", + "model.layers.31.post_attention_layernorm.weight": "pytorch_model-00002-of-00002.bin", + "model.layers.31.self_attn.k_proj.weight": "pytorch_model-00002-of-00002.bin", + "model.layers.31.self_attn.o_proj.weight": "pytorch_model-00002-of-00002.bin", + "model.layers.31.self_attn.q_proj.weight": "pytorch_model-00002-of-00002.bin", + "model.layers.31.self_attn.rotary_emb.inv_freq": "pytorch_model-00002-of-00002.bin", + "model.layers.31.self_attn.v_proj.weight": "pytorch_model-00002-of-00002.bin", + "model.layers.4.input_layernorm.weight": "pytorch_model-00001-of-00002.bin", + "model.layers.4.mlp.down_proj.weight": "pytorch_model-00001-of-00002.bin", + "model.layers.4.mlp.gate_proj.weight": "pytorch_model-00001-of-00002.bin", + "model.layers.4.mlp.up_proj.weight": "pytorch_model-00001-of-00002.bin", + "model.layers.4.post_attention_layernorm.weight": "pytorch_model-00001-of-00002.bin", + "model.layers.4.self_attn.k_proj.weight": "pytorch_model-00001-of-00002.bin", + "model.layers.4.self_attn.o_proj.weight": "pytorch_model-00001-of-00002.bin", + "model.layers.4.self_attn.q_proj.weight": "pytorch_model-00001-of-00002.bin", + "model.layers.4.self_attn.rotary_emb.inv_freq": "pytorch_model-00001-of-00002.bin", + "model.layers.4.self_attn.v_proj.weight": "pytorch_model-00001-of-00002.bin", + "model.layers.5.input_layernorm.weight": "pytorch_model-00001-of-00002.bin", + "model.layers.5.mlp.down_proj.weight": "pytorch_model-00001-of-00002.bin", + "model.layers.5.mlp.gate_proj.weight": "pytorch_model-00001-of-00002.bin", + "model.layers.5.mlp.up_proj.weight": "pytorch_model-00001-of-00002.bin", + "model.layers.5.post_attention_layernorm.weight": "pytorch_model-00001-of-00002.bin", + "model.layers.5.self_attn.k_proj.weight": "pytorch_model-00001-of-00002.bin", + "model.layers.5.self_attn.o_proj.weight": "pytorch_model-00001-of-00002.bin", + "model.layers.5.self_attn.q_proj.weight": "pytorch_model-00001-of-00002.bin", + "model.layers.5.self_attn.rotary_emb.inv_freq": "pytorch_model-00001-of-00002.bin", + "model.layers.5.self_attn.v_proj.weight": "pytorch_model-00001-of-00002.bin", + "model.layers.6.input_layernorm.weight": "pytorch_model-00001-of-00002.bin", + "model.layers.6.mlp.down_proj.weight": "pytorch_model-00001-of-00002.bin", + "model.layers.6.mlp.gate_proj.weight": "pytorch_model-00001-of-00002.bin", + "model.layers.6.mlp.up_proj.weight": "pytorch_model-00001-of-00002.bin", + "model.layers.6.post_attention_layernorm.weight": "pytorch_model-00001-of-00002.bin", + "model.layers.6.self_attn.k_proj.weight": "pytorch_model-00001-of-00002.bin", + "model.layers.6.self_attn.o_proj.weight": "pytorch_model-00001-of-00002.bin", + "model.layers.6.self_attn.q_proj.weight": "pytorch_model-00001-of-00002.bin", + "model.layers.6.self_attn.rotary_emb.inv_freq": "pytorch_model-00001-of-00002.bin", + "model.layers.6.self_attn.v_proj.weight": "pytorch_model-00001-of-00002.bin", + "model.layers.7.input_layernorm.weight": "pytorch_model-00001-of-00002.bin", + "model.layers.7.mlp.down_proj.weight": "pytorch_model-00001-of-00002.bin", + "model.layers.7.mlp.gate_proj.weight": "pytorch_model-00001-of-00002.bin", + "model.layers.7.mlp.up_proj.weight": "pytorch_model-00001-of-00002.bin", + "model.layers.7.post_attention_layernorm.weight": "pytorch_model-00001-of-00002.bin", + "model.layers.7.self_attn.k_proj.weight": "pytorch_model-00001-of-00002.bin", + "model.layers.7.self_attn.o_proj.weight": "pytorch_model-00001-of-00002.bin", + "model.layers.7.self_attn.q_proj.weight": "pytorch_model-00001-of-00002.bin", + "model.layers.7.self_attn.rotary_emb.inv_freq": "pytorch_model-00001-of-00002.bin", + "model.layers.7.self_attn.v_proj.weight": "pytorch_model-00001-of-00002.bin", + "model.layers.8.input_layernorm.weight": "pytorch_model-00001-of-00002.bin", + "model.layers.8.mlp.down_proj.weight": "pytorch_model-00001-of-00002.bin", + "model.layers.8.mlp.gate_proj.weight": "pytorch_model-00001-of-00002.bin", + "model.layers.8.mlp.up_proj.weight": "pytorch_model-00001-of-00002.bin", + "model.layers.8.post_attention_layernorm.weight": "pytorch_model-00001-of-00002.bin", + "model.layers.8.self_attn.k_proj.weight": "pytorch_model-00001-of-00002.bin", + "model.layers.8.self_attn.o_proj.weight": "pytorch_model-00001-of-00002.bin", + "model.layers.8.self_attn.q_proj.weight": "pytorch_model-00001-of-00002.bin", + "model.layers.8.self_attn.rotary_emb.inv_freq": "pytorch_model-00001-of-00002.bin", + "model.layers.8.self_attn.v_proj.weight": "pytorch_model-00001-of-00002.bin", + "model.layers.9.input_layernorm.weight": "pytorch_model-00001-of-00002.bin", + "model.layers.9.mlp.down_proj.weight": "pytorch_model-00001-of-00002.bin", + "model.layers.9.mlp.gate_proj.weight": "pytorch_model-00001-of-00002.bin", + "model.layers.9.mlp.up_proj.weight": "pytorch_model-00001-of-00002.bin", + "model.layers.9.post_attention_layernorm.weight": "pytorch_model-00001-of-00002.bin", + "model.layers.9.self_attn.k_proj.weight": "pytorch_model-00001-of-00002.bin", + "model.layers.9.self_attn.o_proj.weight": "pytorch_model-00001-of-00002.bin", + "model.layers.9.self_attn.q_proj.weight": "pytorch_model-00001-of-00002.bin", + "model.layers.9.self_attn.rotary_emb.inv_freq": "pytorch_model-00001-of-00002.bin", + "model.layers.9.self_attn.v_proj.weight": "pytorch_model-00001-of-00002.bin", + "model.norm.weight": "pytorch_model-00002-of-00002.bin" + } +} diff --git a/special_tokens_map.json b/special_tokens_map.json new file mode 100644 index 0000000..f928b24 --- /dev/null +++ b/special_tokens_map.json @@ -0,0 +1,24 @@ +{ + "bos_token": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false + }, + "eos_token": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false + }, + "pad_token": "", + "unk_token": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false + } +} diff --git a/tokenizer.model b/tokenizer.model new file mode 100644 index 0000000..6c00c74 --- /dev/null +++ b/tokenizer.model @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9e556afd44213b6bd1be2b850ebbbd98f5481437a8021afaf58ee7fb1818d347 +size 499723 diff --git a/tokenizer_config.json b/tokenizer_config.json new file mode 100644 index 0000000..2566c60 --- /dev/null +++ b/tokenizer_config.json @@ -0,0 +1,34 @@ +{ + "add_bos_token": true, + "add_eos_token": false, + "bos_token": { + "__type": "AddedToken", + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false + }, + "clean_up_tokenization_spaces": false, + "eos_token": { + "__type": "AddedToken", + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false + }, + "model_max_length": 1280, + "pad_token": null, + "padding_side": "right", + "sp_model_kwargs": {}, + "tokenizer_class": "LlamaTokenizer", + "unk_token": { + "__type": "AddedToken", + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false + } +} diff --git a/trainer_state.json b/trainer_state.json new file mode 100644 index 0000000..0fe89c5 --- /dev/null +++ b/trainer_state.json @@ -0,0 +1,971 @@ +{ + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 2.9960768928991763, + "global_step": 954, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.03, + "learning_rate": 9.997799572243123e-06, + "loss": 0.4974, + "step": 10 + }, + { + "epoch": 0.06, + "learning_rate": 9.990195641770761e-06, + "loss": 0.4841, + "step": 20 + }, + { + "epoch": 0.06, + "eval_loss": 0.4602764844894409, + "eval_runtime": 62.0223, + "eval_samples_per_second": 13.431, + "eval_steps_per_second": 0.855, + "step": 20 + }, + { + "epoch": 0.09, + "learning_rate": 9.9771693033643e-06, + "loss": 0.4748, + "step": 30 + }, + { + "epoch": 0.13, + "learning_rate": 9.958734711603195e-06, + "loss": 0.4678, + "step": 40 + }, + { + "epoch": 0.13, + "eval_loss": 0.45144811272621155, + "eval_runtime": 61.8568, + "eval_samples_per_second": 13.467, + "eval_steps_per_second": 0.857, + "step": 40 + }, + { + "epoch": 0.16, + "learning_rate": 9.934911897741493e-06, + "loss": 0.4687, + "step": 50 + }, + { + "epoch": 0.19, + "learning_rate": 9.905726747941616e-06, + "loss": 0.4687, + "step": 60 + }, + { + "epoch": 0.19, + "eval_loss": 0.44708773493766785, + "eval_runtime": 62.0907, + "eval_samples_per_second": 13.416, + "eval_steps_per_second": 0.854, + "step": 60 + }, + { + "epoch": 0.22, + "learning_rate": 9.871210975146135e-06, + "loss": 0.473, + "step": 70 + }, + { + "epoch": 0.25, + "learning_rate": 9.831402084618113e-06, + "loss": 0.4608, + "step": 80 + }, + { + "epoch": 0.25, + "eval_loss": 0.44395506381988525, + "eval_runtime": 61.9484, + "eval_samples_per_second": 13.447, + "eval_steps_per_second": 0.856, + "step": 80 + }, + { + "epoch": 0.28, + "learning_rate": 9.786343333187412e-06, + "loss": 0.4542, + "step": 90 + }, + { + "epoch": 0.31, + "learning_rate": 9.736083682247287e-06, + "loss": 0.4593, + "step": 100 + }, + { + "epoch": 0.31, + "eval_loss": 0.43974635004997253, + "eval_runtime": 61.9084, + "eval_samples_per_second": 13.455, + "eval_steps_per_second": 0.856, + "step": 100 + }, + { + "epoch": 0.35, + "learning_rate": 9.680677744552346e-06, + "loss": 0.4681, + "step": 110 + }, + { + "epoch": 0.38, + "learning_rate": 9.620185724875652e-06, + "loss": 0.4488, + "step": 120 + }, + { + "epoch": 0.38, + "eval_loss": 0.435758113861084, + "eval_runtime": 61.8043, + "eval_samples_per_second": 13.478, + "eval_steps_per_second": 0.858, + "step": 120 + }, + { + "epoch": 0.41, + "learning_rate": 9.55467335458948e-06, + "loss": 0.4478, + "step": 130 + }, + { + "epoch": 0.44, + "learning_rate": 9.484211820240797e-06, + "loss": 0.4494, + "step": 140 + }, + { + "epoch": 0.44, + "eval_loss": 0.434115469455719, + "eval_runtime": 61.841, + "eval_samples_per_second": 13.47, + "eval_steps_per_second": 0.857, + "step": 140 + }, + { + "epoch": 0.47, + "learning_rate": 9.408877686199078e-06, + "loss": 0.44, + "step": 150 + }, + { + "epoch": 0.5, + "learning_rate": 9.328752811460542e-06, + "loss": 0.4477, + "step": 160 + }, + { + "epoch": 0.5, + "eval_loss": 0.4319334030151367, + "eval_runtime": 61.9956, + "eval_samples_per_second": 13.436, + "eval_steps_per_second": 0.855, + "step": 160 + }, + { + "epoch": 0.53, + "learning_rate": 9.243924260699133e-06, + "loss": 0.4465, + "step": 170 + }, + { + "epoch": 0.56, + "learning_rate": 9.154484209661002e-06, + "loss": 0.445, + "step": 180 + }, + { + "epoch": 0.56, + "eval_loss": 0.42997926473617554, + "eval_runtime": 62.0149, + "eval_samples_per_second": 13.432, + "eval_steps_per_second": 0.855, + "step": 180 + }, + { + "epoch": 0.6, + "learning_rate": 9.060529845005184e-06, + "loss": 0.4332, + "step": 190 + }, + { + "epoch": 0.63, + "learning_rate": 8.962163258699397e-06, + "loss": 0.4366, + "step": 200 + }, + { + "epoch": 0.63, + "eval_loss": 0.4274918735027313, + "eval_runtime": 61.8797, + "eval_samples_per_second": 13.462, + "eval_steps_per_second": 0.857, + "step": 200 + }, + { + "epoch": 0.66, + "learning_rate": 8.859491337085643e-06, + "loss": 0.428, + "step": 210 + }, + { + "epoch": 0.69, + "learning_rate": 8.752625644736204e-06, + "loss": 0.442, + "step": 220 + }, + { + "epoch": 0.69, + "eval_loss": 0.42491650581359863, + "eval_runtime": 62.1183, + "eval_samples_per_second": 13.41, + "eval_steps_per_second": 0.853, + "step": 220 + }, + { + "epoch": 0.72, + "learning_rate": 8.641682303226197e-06, + "loss": 0.4442, + "step": 230 + }, + { + "epoch": 0.75, + "learning_rate": 8.526781864954453e-06, + "loss": 0.4424, + "step": 240 + }, + { + "epoch": 0.75, + "eval_loss": 0.4228505492210388, + "eval_runtime": 62.0338, + "eval_samples_per_second": 13.428, + "eval_steps_per_second": 0.854, + "step": 240 + }, + { + "epoch": 0.78, + "learning_rate": 8.40804918214979e-06, + "loss": 0.4301, + "step": 250 + }, + { + "epoch": 0.82, + "learning_rate": 8.28561327120505e-06, + "loss": 0.4427, + "step": 260 + }, + { + "epoch": 0.82, + "eval_loss": 0.41998758912086487, + "eval_runtime": 61.9344, + "eval_samples_per_second": 13.45, + "eval_steps_per_second": 0.856, + "step": 260 + }, + { + "epoch": 0.85, + "learning_rate": 8.159607172486301e-06, + "loss": 0.4316, + "step": 270 + }, + { + "epoch": 0.88, + "learning_rate": 8.030167805769537e-06, + "loss": 0.4372, + "step": 280 + }, + { + "epoch": 0.88, + "eval_loss": 0.4175536036491394, + "eval_runtime": 61.971, + "eval_samples_per_second": 13.442, + "eval_steps_per_second": 0.855, + "step": 280 + }, + { + "epoch": 0.91, + "learning_rate": 7.897435821461964e-06, + "loss": 0.4398, + "step": 290 + }, + { + "epoch": 0.94, + "learning_rate": 7.761555447769548e-06, + "loss": 0.4335, + "step": 300 + }, + { + "epoch": 0.94, + "eval_loss": 0.4155929982662201, + "eval_runtime": 61.9696, + "eval_samples_per_second": 13.442, + "eval_steps_per_second": 0.855, + "step": 300 + }, + { + "epoch": 0.97, + "learning_rate": 7.622674333976863e-06, + "loss": 0.4371, + "step": 310 + }, + { + "epoch": 1.0, + "learning_rate": 7.4809433900095705e-06, + "loss": 0.4088, + "step": 320 + }, + { + "epoch": 1.0, + "eval_loss": 0.41465404629707336, + "eval_runtime": 61.9822, + "eval_samples_per_second": 13.439, + "eval_steps_per_second": 0.855, + "step": 320 + }, + { + "epoch": 1.04, + "learning_rate": 7.336516622453833e-06, + "loss": 0.3166, + "step": 330 + }, + { + "epoch": 1.07, + "learning_rate": 7.1895509672108674e-06, + "loss": 0.3145, + "step": 340 + }, + { + "epoch": 1.07, + "eval_loss": 0.42051395773887634, + "eval_runtime": 62.5118, + "eval_samples_per_second": 13.325, + "eval_steps_per_second": 0.848, + "step": 340 + }, + { + "epoch": 1.1, + "learning_rate": 7.040206118968466e-06, + "loss": 0.3136, + "step": 350 + }, + { + "epoch": 1.13, + "learning_rate": 6.88864435767478e-06, + "loss": 0.3151, + "step": 360 + }, + { + "epoch": 1.13, + "eval_loss": 0.4205115735530853, + "eval_runtime": 62.5792, + "eval_samples_per_second": 13.311, + "eval_steps_per_second": 0.847, + "step": 360 + }, + { + "epoch": 1.16, + "learning_rate": 6.735030372202942e-06, + "loss": 0.3137, + "step": 370 + }, + { + "epoch": 1.19, + "learning_rate": 6.579531081398105e-06, + "loss": 0.3019, + "step": 380 + }, + { + "epoch": 1.19, + "eval_loss": 0.4216003119945526, + "eval_runtime": 62.3646, + "eval_samples_per_second": 13.357, + "eval_steps_per_second": 0.85, + "step": 380 + }, + { + "epoch": 1.22, + "learning_rate": 6.4223154527013755e-06, + "loss": 0.3044, + "step": 390 + }, + { + "epoch": 1.26, + "learning_rate": 6.263554318547713e-06, + "loss": 0.3044, + "step": 400 + }, + { + "epoch": 1.26, + "eval_loss": 0.4185173809528351, + "eval_runtime": 61.9058, + "eval_samples_per_second": 13.456, + "eval_steps_per_second": 0.856, + "step": 400 + }, + { + "epoch": 1.29, + "learning_rate": 6.1034201907373045e-06, + "loss": 0.305, + "step": 410 + }, + { + "epoch": 1.32, + "learning_rate": 5.942087072982131e-06, + "loss": 0.3034, + "step": 420 + }, + { + "epoch": 1.32, + "eval_loss": 0.41815003752708435, + "eval_runtime": 62.0238, + "eval_samples_per_second": 13.43, + "eval_steps_per_second": 0.855, + "step": 420 + }, + { + "epoch": 1.35, + "learning_rate": 5.779730271831384e-06, + "loss": 0.3115, + "step": 430 + }, + { + "epoch": 1.38, + "learning_rate": 5.616526206181215e-06, + "loss": 0.3026, + "step": 440 + }, + { + "epoch": 1.38, + "eval_loss": 0.41711267828941345, + "eval_runtime": 62.5134, + "eval_samples_per_second": 13.325, + "eval_steps_per_second": 0.848, + "step": 440 + }, + { + "epoch": 1.41, + "learning_rate": 5.4526522155758015e-06, + "loss": 0.3077, + "step": 450 + }, + { + "epoch": 1.45, + "learning_rate": 5.288286367508009e-06, + "loss": 0.3062, + "step": 460 + }, + { + "epoch": 1.45, + "eval_loss": 0.41751572489738464, + "eval_runtime": 62.561, + "eval_samples_per_second": 13.315, + "eval_steps_per_second": 0.847, + "step": 460 + }, + { + "epoch": 1.48, + "learning_rate": 5.123607263929075e-06, + "loss": 0.3076, + "step": 470 + }, + { + "epoch": 1.51, + "learning_rate": 4.958793847177518e-06, + "loss": 0.315, + "step": 480 + }, + { + "epoch": 1.51, + "eval_loss": 0.41455498337745667, + "eval_runtime": 62.0669, + "eval_samples_per_second": 13.421, + "eval_steps_per_second": 0.854, + "step": 480 + }, + { + "epoch": 1.54, + "learning_rate": 4.7940252055382115e-06, + "loss": 0.3024, + "step": 490 + }, + { + "epoch": 1.57, + "learning_rate": 4.629480378642832e-06, + "loss": 0.3124, + "step": 500 + }, + { + "epoch": 1.57, + "eval_loss": 0.41453319787979126, + "eval_runtime": 61.9067, + "eval_samples_per_second": 13.456, + "eval_steps_per_second": 0.856, + "step": 500 + }, + { + "epoch": 1.6, + "learning_rate": 4.46533816292321e-06, + "loss": 0.31, + "step": 510 + }, + { + "epoch": 1.63, + "learning_rate": 4.301776917328918e-06, + "loss": 0.3096, + "step": 520 + }, + { + "epoch": 1.63, + "eval_loss": 0.41273748874664307, + "eval_runtime": 62.1417, + "eval_samples_per_second": 13.405, + "eval_steps_per_second": 0.853, + "step": 520 + }, + { + "epoch": 1.67, + "learning_rate": 4.138974369520252e-06, + "loss": 0.3044, + "step": 530 + }, + { + "epoch": 1.7, + "learning_rate": 3.977107422747163e-06, + "loss": 0.3178, + "step": 540 + }, + { + "epoch": 1.7, + "eval_loss": 0.4111482501029968, + "eval_runtime": 62.5639, + "eval_samples_per_second": 13.314, + "eval_steps_per_second": 0.847, + "step": 540 + }, + { + "epoch": 1.73, + "learning_rate": 3.816351963624017e-06, + "loss": 0.3102, + "step": 550 + }, + { + "epoch": 1.76, + "learning_rate": 3.6568826710090353e-06, + "loss": 0.3044, + "step": 560 + }, + { + "epoch": 1.76, + "eval_loss": 0.4110707640647888, + "eval_runtime": 62.5689, + "eval_samples_per_second": 13.313, + "eval_steps_per_second": 0.847, + "step": 560 + }, + { + "epoch": 1.79, + "learning_rate": 3.4988728261960957e-06, + "loss": 0.306, + "step": 570 + }, + { + "epoch": 1.82, + "learning_rate": 3.3424941246251574e-06, + "loss": 0.3078, + "step": 580 + }, + { + "epoch": 1.82, + "eval_loss": 0.4091060757637024, + "eval_runtime": 62.0676, + "eval_samples_per_second": 13.421, + "eval_steps_per_second": 0.854, + "step": 580 + }, + { + "epoch": 1.85, + "learning_rate": 3.1879164893158713e-06, + "loss": 0.2977, + "step": 590 + }, + { + "epoch": 1.89, + "learning_rate": 3.035307886227156e-06, + "loss": 0.2967, + "step": 600 + }, + { + "epoch": 1.89, + "eval_loss": 0.4094270169734955, + "eval_runtime": 62.0655, + "eval_samples_per_second": 13.421, + "eval_steps_per_second": 0.854, + "step": 600 + }, + { + "epoch": 1.92, + "learning_rate": 2.8848341417433036e-06, + "loss": 0.3069, + "step": 610 + }, + { + "epoch": 1.95, + "learning_rate": 2.736658762485005e-06, + "loss": 0.3068, + "step": 620 + }, + { + "epoch": 1.95, + "eval_loss": 0.4080323278903961, + "eval_runtime": 62.0098, + "eval_samples_per_second": 13.433, + "eval_steps_per_second": 0.855, + "step": 620 + }, + { + "epoch": 1.98, + "learning_rate": 2.590942757641035e-06, + "loss": 0.3037, + "step": 630 + }, + { + "epoch": 2.01, + "learning_rate": 2.447844464013703e-06, + "loss": 0.276, + "step": 640 + }, + { + "epoch": 2.01, + "eval_loss": 0.4152510464191437, + "eval_runtime": 61.9609, + "eval_samples_per_second": 13.444, + "eval_steps_per_second": 0.855, + "step": 640 + }, + { + "epoch": 2.04, + "learning_rate": 2.3075193739681182e-06, + "loss": 0.2327, + "step": 650 + }, + { + "epoch": 2.07, + "learning_rate": 2.170119966472293e-06, + "loss": 0.2288, + "step": 660 + }, + { + "epoch": 2.07, + "eval_loss": 0.43204566836357117, + "eval_runtime": 62.0792, + "eval_samples_per_second": 13.418, + "eval_steps_per_second": 0.854, + "step": 660 + }, + { + "epoch": 2.1, + "learning_rate": 2.0357955414116075e-06, + "loss": 0.2267, + "step": 670 + }, + { + "epoch": 2.14, + "learning_rate": 1.9046920573577239e-06, + "loss": 0.2244, + "step": 680 + }, + { + "epoch": 2.14, + "eval_loss": 0.4292474389076233, + "eval_runtime": 62.0223, + "eval_samples_per_second": 13.431, + "eval_steps_per_second": 0.855, + "step": 680 + }, + { + "epoch": 2.17, + "learning_rate": 1.7769519729682105e-06, + "loss": 0.2327, + "step": 690 + }, + { + "epoch": 2.2, + "learning_rate": 1.6527140921892066e-06, + "loss": 0.2336, + "step": 700 + }, + { + "epoch": 2.2, + "eval_loss": 0.427610844373703, + "eval_runtime": 62.1107, + "eval_samples_per_second": 13.412, + "eval_steps_per_second": 0.853, + "step": 700 + }, + { + "epoch": 2.23, + "learning_rate": 1.532113413429357e-06, + "loss": 0.2386, + "step": 710 + }, + { + "epoch": 2.26, + "learning_rate": 1.4152809828688708e-06, + "loss": 0.2266, + "step": 720 + }, + { + "epoch": 2.26, + "eval_loss": 0.4290391206741333, + "eval_runtime": 62.2449, + "eval_samples_per_second": 13.383, + "eval_steps_per_second": 0.851, + "step": 720 + }, + { + "epoch": 2.29, + "learning_rate": 1.3023437520631426e-06, + "loss": 0.2328, + "step": 730 + }, + { + "epoch": 2.32, + "learning_rate": 1.1934244399956206e-06, + "loss": 0.2312, + "step": 740 + }, + { + "epoch": 2.32, + "eval_loss": 0.42950907349586487, + "eval_runtime": 62.2915, + "eval_samples_per_second": 13.373, + "eval_steps_per_second": 0.851, + "step": 740 + }, + { + "epoch": 2.36, + "learning_rate": 1.0886413997298595e-06, + "loss": 0.2338, + "step": 750 + }, + { + "epoch": 2.39, + "learning_rate": 9.881084898056197e-07, + "loss": 0.2277, + "step": 760 + }, + { + "epoch": 2.39, + "eval_loss": 0.4284292161464691, + "eval_runtime": 62.4097, + "eval_samples_per_second": 13.347, + "eval_steps_per_second": 0.849, + "step": 760 + }, + { + "epoch": 2.42, + "learning_rate": 8.919349505187813e-07, + "loss": 0.2333, + "step": 770 + }, + { + "epoch": 2.45, + "learning_rate": 8.002252852194992e-07, + "loss": 0.2332, + "step": 780 + }, + { + "epoch": 2.45, + "eval_loss": 0.42790091037750244, + "eval_runtime": 62.4651, + "eval_samples_per_second": 13.335, + "eval_steps_per_second": 0.848, + "step": 780 + }, + { + "epoch": 2.48, + "learning_rate": 7.130791467575676e-07, + "loss": 0.2257, + "step": 790 + }, + { + "epoch": 2.51, + "learning_rate": 6.305912291984229e-07, + "loss": 0.2289, + "step": 800 + }, + { + "epoch": 2.51, + "eval_loss": 0.42792582511901855, + "eval_runtime": 62.5296, + "eval_samples_per_second": 13.322, + "eval_steps_per_second": 0.848, + "step": 800 + }, + { + "epoch": 2.54, + "learning_rate": 5.528511649273932e-07, + "loss": 0.2303, + "step": 810 + }, + { + "epoch": 2.58, + "learning_rate": 4.799434272540576e-07, + "loss": 0.2279, + "step": 820 + }, + { + "epoch": 2.58, + "eval_loss": 0.4278266131877899, + "eval_runtime": 62.5218, + "eval_samples_per_second": 13.323, + "eval_steps_per_second": 0.848, + "step": 820 + }, + { + "epoch": 2.61, + "learning_rate": 4.1194723862250317e-07, + "loss": 0.2267, + "step": 830 + }, + { + "epoch": 2.64, + "learning_rate": 3.4893648452724636e-07, + "loss": 0.2312, + "step": 840 + }, + { + "epoch": 2.64, + "eval_loss": 0.4273243546485901, + "eval_runtime": 62.5591, + "eval_samples_per_second": 13.315, + "eval_steps_per_second": 0.847, + "step": 840 + }, + { + "epoch": 2.67, + "learning_rate": 2.9097963322834597e-07, + "loss": 0.2306, + "step": 850 + }, + { + "epoch": 2.7, + "learning_rate": 2.3813966135294574e-07, + "loss": 0.2334, + "step": 860 + }, + { + "epoch": 2.7, + "eval_loss": 0.42646506428718567, + "eval_runtime": 62.6378, + "eval_samples_per_second": 13.299, + "eval_steps_per_second": 0.846, + "step": 860 + }, + { + "epoch": 2.73, + "learning_rate": 1.9047398546410633e-07, + "loss": 0.2306, + "step": 870 + }, + { + "epoch": 2.76, + "learning_rate": 1.4803439967125022e-07, + "loss": 0.2278, + "step": 880 + }, + { + "epoch": 2.76, + "eval_loss": 0.42754805088043213, + "eval_runtime": 62.1476, + "eval_samples_per_second": 13.404, + "eval_steps_per_second": 0.853, + "step": 880 + }, + { + "epoch": 2.8, + "learning_rate": 1.1086701935005606e-07, + "loss": 0.2296, + "step": 890 + }, + { + "epoch": 2.83, + "learning_rate": 7.901223103291833e-08, + "loss": 0.2295, + "step": 900 + }, + { + "epoch": 2.83, + "eval_loss": 0.4276488721370697, + "eval_runtime": 61.9957, + "eval_samples_per_second": 13.436, + "eval_steps_per_second": 0.855, + "step": 900 + }, + { + "epoch": 2.86, + "learning_rate": 5.250464852444792e-08, + "loss": 0.2334, + "step": 910 + }, + { + "epoch": 2.89, + "learning_rate": 3.137307528968292e-08, + "loss": 0.2292, + "step": 920 + }, + { + "epoch": 2.89, + "eval_loss": 0.4273829162120819, + "eval_runtime": 62.0452, + "eval_samples_per_second": 13.426, + "eval_steps_per_second": 0.854, + "step": 920 + }, + { + "epoch": 2.92, + "learning_rate": 1.5640473155894566e-08, + "loss": 0.2284, + "step": 930 + }, + { + "epoch": 2.95, + "learning_rate": 5.323937361977338e-09, + "loss": 0.2291, + "step": 940 + }, + { + "epoch": 2.95, + "eval_loss": 0.42734310030937195, + "eval_runtime": 61.9825, + "eval_samples_per_second": 13.439, + "eval_steps_per_second": 0.855, + "step": 940 + }, + { + "epoch": 2.98, + "learning_rate": 4.346779825575853e-10, + "loss": 0.2288, + "step": 950 + }, + { + "epoch": 3.0, + "step": 954, + "total_flos": 6.195687991759864e+18, + "train_loss": 0.1527079766776327, + "train_runtime": 29559.5606, + "train_samples_per_second": 4.138, + "train_steps_per_second": 0.032 + } + ], + "max_steps": 954, + "num_train_epochs": 3, + "total_flos": 6.195687991759864e+18, + "trial_name": null, + "trial_params": null +}