初始化项目，由ModelHub XC社区提供模型
Model: ifable/gemma-2-Ifable-9B Source: Original Platform
This commit is contained in:
ModelHub XC
36
.gitattributes
vendored
Normal file
36
.gitattributes
vendored
Normal file
@@ -0,0 +1,36 @@
|
||||
*.7z filter=lfs diff=lfs merge=lfs -text
|
||||
*.arrow filter=lfs diff=lfs merge=lfs -text
|
||||
*.bin filter=lfs diff=lfs merge=lfs -text
|
||||
*.bz2 filter=lfs diff=lfs merge=lfs -text
|
||||
*.ckpt filter=lfs diff=lfs merge=lfs -text
|
||||
*.ftz filter=lfs diff=lfs merge=lfs -text
|
||||
*.gz filter=lfs diff=lfs merge=lfs -text
|
||||
*.h5 filter=lfs diff=lfs merge=lfs -text
|
||||
*.joblib filter=lfs diff=lfs merge=lfs -text
|
||||
*.lfs.* filter=lfs diff=lfs merge=lfs -text
|
||||
*.mlmodel filter=lfs diff=lfs merge=lfs -text
|
||||
*.model filter=lfs diff=lfs merge=lfs -text
|
||||
*.msgpack filter=lfs diff=lfs merge=lfs -text
|
||||
*.npy filter=lfs diff=lfs merge=lfs -text
|
||||
*.npz filter=lfs diff=lfs merge=lfs -text
|
||||
*.onnx filter=lfs diff=lfs merge=lfs -text
|
||||
*.ot filter=lfs diff=lfs merge=lfs -text
|
||||
*.parquet filter=lfs diff=lfs merge=lfs -text
|
||||
*.pb filter=lfs diff=lfs merge=lfs -text
|
||||
*.pickle filter=lfs diff=lfs merge=lfs -text
|
||||
*.pkl filter=lfs diff=lfs merge=lfs -text
|
||||
*.pt filter=lfs diff=lfs merge=lfs -text
|
||||
*.pth filter=lfs diff=lfs merge=lfs -text
|
||||
*.rar filter=lfs diff=lfs merge=lfs -text
|
||||
*.safetensors filter=lfs diff=lfs merge=lfs -text
|
||||
saved_model/**/* filter=lfs diff=lfs merge=lfs -text
|
||||
*.tar.* filter=lfs diff=lfs merge=lfs -text
|
||||
*.tar filter=lfs diff=lfs merge=lfs -text
|
||||
*.tflite filter=lfs diff=lfs merge=lfs -text
|
||||
*.tgz filter=lfs diff=lfs merge=lfs -text
|
||||
*.wasm filter=lfs diff=lfs merge=lfs -text
|
||||
*.xz filter=lfs diff=lfs merge=lfs -text
|
||||
*.zip filter=lfs diff=lfs merge=lfs -text
|
||||
*.zst filter=lfs diff=lfs merge=lfs -text
|
||||
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
||||
tokenizer.json filter=lfs diff=lfs merge=lfs -text
|
||||
65
README.md
Normal file
65
README.md
Normal file
@@ -0,0 +1,65 @@
|
||||
---
|
||||
license: gemma
|
||||
library_name: transformers
|
||||
datasets:
|
||||
- jondurbin/gutenberg-dpo-v0.1
|
||||
---
|
||||
<!-- This model card has been generated automatically according to the information the Trainer had access to. You
|
||||
should probably proofread and complete it, then remove this comment. -->
|
||||
|
||||
# ifable/gemma-2-Ifable-9B
|
||||
This model ranked first on the Creative Writing Benchmark (https://eqbench.com/creative_writing.html) on September 10, 2024
|
||||
|
||||
## Training and evaluation data
|
||||
|
||||
- Gutenberg: https://huggingface.co/datasets/jondurbin/gutenberg-dpo-v0.1
|
||||
- Carefully curated proprietary creative writing dataset
|
||||
|
||||
## Training procedure
|
||||
|
||||
Training method: SimPO (GitHub - princeton-nlp/SimPO: SimPO: Simple Preference Optimization with a Reference-Free Reward)
|
||||
|
||||
It achieves the following results on the evaluation set:
|
||||
- Loss: 1.0163
|
||||
- Rewards/chosen: -21.6822
|
||||
- Rewards/rejected: -47.8754
|
||||
- Rewards/accuracies: 0.9167
|
||||
- Rewards/margins: 26.1931
|
||||
- Logps/rejected: -4.7875
|
||||
- Logps/chosen: -2.1682
|
||||
- Logits/rejected: -17.0475
|
||||
- Logits/chosen: -12.0041
|
||||
|
||||
### Training hyperparameters
|
||||
|
||||
The following hyperparameters were used during training:
|
||||
- learning_rate: 8e-07
|
||||
- train_batch_size: 1
|
||||
- eval_batch_size: 1
|
||||
- seed: 42
|
||||
- distributed_type: multi-GPU
|
||||
- num_devices: 8
|
||||
- gradient_accumulation_steps: 16
|
||||
- total_train_batch_size: 128
|
||||
- total_eval_batch_size: 8
|
||||
- optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
|
||||
- lr_scheduler_type: cosine
|
||||
- lr_scheduler_warmup_ratio: 0.1
|
||||
- num_epochs: 1.0
|
||||
|
||||
### Training results
|
||||
|
||||
| Training Loss | Epoch | Step | Validation Loss | Rewards/chosen | Rewards/rejected | Rewards/accuracies | Rewards/margins | Logps/rejected | Logps/chosen | Logits/rejected | Logits/chosen | Sft Loss |
|
||||
|:-------------:|:------:|:----:|:---------------:|:--------------:|:----------------:|:------------------:|:---------------:|:--------------:|:------------:|:---------------:|:-------------:|:--------:|
|
||||
| 1.4444 | 0.9807 | 35 | 1.0163 | -21.6822 | -47.8754 | 0.9167 | 26.1931 | -4.7875 | -2.1682 | -17.0475 | -12.0041 | 0.0184 |
|
||||
|
||||
|
||||
### Framework versions
|
||||
|
||||
- Transformers 4.43.4
|
||||
- Pytorch 2.3.0a0+ebedce2
|
||||
- Datasets 2.20.0
|
||||
- Tokenizers 0.19.1
|
||||
|
||||
|
||||
We are looking for product manager and operations managers to build applications through our model, and also open for business cooperation, and also AI engineer to join us, contact with : contact@ifable.ai
|
||||
35
all_results.json
Normal file
35
all_results.json
Normal file
@@ -0,0 +1,35 @@
|
||||
{
|
||||
"before_init_mem_cpu": 3802071040,
|
||||
"before_init_mem_gpu": 22016,
|
||||
"epoch": 0.9807355516637478,
|
||||
"eval_logits/chosen": -12.004097938537598,
|
||||
"eval_logits/rejected": -17.047502517700195,
|
||||
"eval_logps/chosen": -2.168222427368164,
|
||||
"eval_logps/rejected": -4.787535667419434,
|
||||
"eval_loss": 1.0162526369094849,
|
||||
"eval_mem_cpu_alloc_delta": 466944,
|
||||
"eval_mem_cpu_peaked_delta": 0,
|
||||
"eval_mem_gpu_alloc_delta": 0,
|
||||
"eval_mem_gpu_peaked_delta": 25220711424,
|
||||
"eval_rewards/accuracies": 0.9166666865348816,
|
||||
"eval_rewards/chosen": -21.682226181030273,
|
||||
"eval_rewards/margins": 26.193130493164062,
|
||||
"eval_rewards/rejected": -47.875362396240234,
|
||||
"eval_runtime": 9.9413,
|
||||
"eval_samples_per_second": 9.456,
|
||||
"eval_sft_loss": 0.01844729855656624,
|
||||
"eval_steps_per_second": 1.207,
|
||||
"init_mem_cpu_alloc_delta": 364544,
|
||||
"init_mem_cpu_peaked_delta": 0,
|
||||
"init_mem_gpu_alloc_delta": 0,
|
||||
"init_mem_gpu_peaked_delta": 0,
|
||||
"total_flos": 39867492466688.0,
|
||||
"train_loss": 3.085822834287371,
|
||||
"train_mem_cpu_alloc_delta": 5213659136,
|
||||
"train_mem_cpu_peaked_delta": 22737326080,
|
||||
"train_mem_gpu_alloc_delta": 16267848704,
|
||||
"train_mem_gpu_peaked_delta": 36029468160,
|
||||
"train_runtime": 1628.7465,
|
||||
"train_samples_per_second": 2.805,
|
||||
"train_steps_per_second": 0.021
|
||||
}
|
||||
34
config.json
Normal file
34
config.json
Normal file
@@ -0,0 +1,34 @@
|
||||
{
|
||||
"_name_or_path": "ifable/gemma-2-Ifable-9B",
|
||||
"architectures": [
|
||||
"Gemma2ForCausalLM"
|
||||
],
|
||||
"attention_bias": false,
|
||||
"attention_dropout": 0.0,
|
||||
"attn_logit_softcapping": 50.0,
|
||||
"bos_token_id": 2,
|
||||
"cache_implementation": "hybrid",
|
||||
"eos_token_id": 1,
|
||||
"final_logit_softcapping": 30.0,
|
||||
"head_dim": 256,
|
||||
"hidden_act": "gelu_pytorch_tanh",
|
||||
"hidden_activation": "gelu_pytorch_tanh",
|
||||
"hidden_size": 3584,
|
||||
"initializer_range": 0.02,
|
||||
"intermediate_size": 14336,
|
||||
"max_position_embeddings": 8192,
|
||||
"model_type": "gemma2",
|
||||
"num_attention_heads": 16,
|
||||
"num_hidden_layers": 42,
|
||||
"num_key_value_heads": 8,
|
||||
"pad_token_id": 0,
|
||||
"query_pre_attn_scalar": 256,
|
||||
"rms_norm_eps": 1e-06,
|
||||
"rope_theta": 10000.0,
|
||||
"sliding_window": 4096,
|
||||
"sliding_window_size": 4096,
|
||||
"torch_dtype": "bfloat16",
|
||||
"transformers_version": "4.43.4",
|
||||
"use_cache": false,
|
||||
"vocab_size": 256000
|
||||
}
|
||||
20
eval_results.json
Normal file
20
eval_results.json
Normal file
@@ -0,0 +1,20 @@
|
||||
{
|
||||
"epoch": 0.9807355516637478,
|
||||
"eval_logits/chosen": -12.004097938537598,
|
||||
"eval_logits/rejected": -17.047502517700195,
|
||||
"eval_logps/chosen": -2.168222427368164,
|
||||
"eval_logps/rejected": -4.787535667419434,
|
||||
"eval_loss": 1.0162526369094849,
|
||||
"eval_mem_cpu_alloc_delta": 466944,
|
||||
"eval_mem_cpu_peaked_delta": 0,
|
||||
"eval_mem_gpu_alloc_delta": 0,
|
||||
"eval_mem_gpu_peaked_delta": 25220711424,
|
||||
"eval_rewards/accuracies": 0.9166666865348816,
|
||||
"eval_rewards/chosen": -21.682226181030273,
|
||||
"eval_rewards/margins": 26.193130493164062,
|
||||
"eval_rewards/rejected": -47.875362396240234,
|
||||
"eval_runtime": 9.9413,
|
||||
"eval_samples_per_second": 9.456,
|
||||
"eval_sft_loss": 0.01844729855656624,
|
||||
"eval_steps_per_second": 1.207
|
||||
}
|
||||
11
generation_config.json
Normal file
11
generation_config.json
Normal file
@@ -0,0 +1,11 @@
|
||||
{
|
||||
"_from_model_config": true,
|
||||
"bos_token_id": 2,
|
||||
"cache_implementation": "hybrid",
|
||||
"eos_token_id": [
|
||||
1,
|
||||
107
|
||||
],
|
||||
"pad_token_id": 0,
|
||||
"transformers_version": "4.43.4"
|
||||
}
|
||||
3
model-00001-of-00004.safetensors
Normal file
3
model-00001-of-00004.safetensors
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:29e54c34e5b5469e4e8395e22b5ef2ed247aef905e4d0e13c9c79e00a7187689
|
||||
size 4903351912
|
||||
3
model-00002-of-00004.safetensors
Normal file
3
model-00002-of-00004.safetensors
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:9e4498718a888d98cff642c267bc5560fa9bc7824f9f0df93cd1af9b573b4cab
|
||||
size 4947570872
|
||||
3
model-00003-of-00004.safetensors
Normal file
3
model-00003-of-00004.safetensors
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:c11acf91db33a207fd3589b750265820c46557a2545f8148bfbd518cfb6d3641
|
||||
size 4962221464
|
||||
3
model-00004-of-00004.safetensors
Normal file
3
model-00004-of-00004.safetensors
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:1227c9f06b027775a279ae353cc4d504617ee9d8a5f0cbae05c261029c607316
|
||||
size 3670322200
|
||||
471
model.safetensors.index.json
Normal file
471
model.safetensors.index.json
Normal file
@@ -0,0 +1,471 @@
|
||||
{
|
||||
"metadata": {
|
||||
"total_size": 18483411968
|
||||
},
|
||||
"weight_map": {
|
||||
"model.embed_tokens.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.0.input_layernorm.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.0.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.0.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.0.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.0.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.0.post_feedforward_layernorm.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.0.pre_feedforward_layernorm.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.0.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.0.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.0.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.0.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.1.input_layernorm.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.1.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.1.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.1.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.1.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.1.post_feedforward_layernorm.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.1.pre_feedforward_layernorm.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.1.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.1.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.1.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.1.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.10.input_layernorm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.10.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.10.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.10.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.10.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.10.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.10.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.10.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.10.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.10.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.10.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.11.input_layernorm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.11.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.11.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.11.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.11.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.11.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.11.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.11.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.11.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.11.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.11.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.12.input_layernorm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.12.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.12.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.12.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.12.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.12.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.12.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.12.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.12.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.12.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.12.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.13.input_layernorm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.13.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.13.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.13.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.13.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.13.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.13.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.13.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.13.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.13.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.13.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.14.input_layernorm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.14.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.14.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.14.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.14.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.14.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.14.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.14.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.14.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.14.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.14.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.15.input_layernorm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.15.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.15.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.15.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.15.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.15.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.15.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.15.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.15.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.15.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.15.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.16.input_layernorm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.16.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.16.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.16.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.16.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.16.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.16.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.16.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.16.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.16.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.16.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.17.input_layernorm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.17.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.17.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.17.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.17.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.17.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.17.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.17.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.17.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.17.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.17.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.18.input_layernorm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.18.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.18.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.18.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.18.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.18.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.18.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.18.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.18.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.18.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.18.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.19.input_layernorm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.19.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.19.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.19.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.19.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.19.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.19.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.19.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.19.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.19.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.19.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.2.input_layernorm.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.2.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.2.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.2.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.2.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.2.post_feedforward_layernorm.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.2.pre_feedforward_layernorm.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.2.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.2.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.2.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.2.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.20.input_layernorm.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.20.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.20.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.20.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.20.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.20.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.20.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.20.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.20.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.20.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.20.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.21.input_layernorm.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.21.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.21.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.21.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.21.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.21.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.21.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.21.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.21.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.21.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.21.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.22.input_layernorm.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.22.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.22.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.22.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.22.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.22.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.22.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.22.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.22.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.22.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.22.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.23.input_layernorm.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.23.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.23.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.23.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.23.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.23.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.23.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.23.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.23.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.23.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.23.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.24.input_layernorm.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.24.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.24.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.24.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.24.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.24.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.24.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.24.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.24.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.24.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.24.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.25.input_layernorm.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.25.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.25.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.25.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.25.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.25.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.25.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.25.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.25.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.25.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.25.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.26.input_layernorm.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.26.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.26.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.26.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.26.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.26.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.26.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.26.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.26.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.26.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.26.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.27.input_layernorm.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.27.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.27.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.27.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.27.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.27.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.27.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.27.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.27.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.27.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.27.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.28.input_layernorm.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.28.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.28.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.28.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.28.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.28.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.28.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.28.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.28.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.28.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.28.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.29.input_layernorm.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.29.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.29.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.29.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.29.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.29.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.29.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.29.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.29.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.29.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.29.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.3.input_layernorm.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.3.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.3.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.3.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.3.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.3.post_feedforward_layernorm.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.3.pre_feedforward_layernorm.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.3.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.3.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.3.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.3.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.30.input_layernorm.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.30.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.30.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.30.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.30.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.30.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.30.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.30.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.30.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.30.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.30.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.31.input_layernorm.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.31.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.31.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.31.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.31.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.31.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.31.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.31.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.31.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.31.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.31.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.32.input_layernorm.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.32.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.32.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.32.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.32.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.32.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.32.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.32.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.32.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.32.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.32.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.33.input_layernorm.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.33.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.33.mlp.gate_proj.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.33.mlp.up_proj.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.33.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.33.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.33.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.33.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.33.self_attn.o_proj.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.33.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.33.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.34.input_layernorm.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.34.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.34.mlp.gate_proj.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.34.mlp.up_proj.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.34.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.34.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.34.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.34.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.34.self_attn.o_proj.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.34.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.34.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.35.input_layernorm.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.35.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.35.mlp.gate_proj.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.35.mlp.up_proj.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.35.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.35.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.35.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.35.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.35.self_attn.o_proj.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.35.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.35.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.36.input_layernorm.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.36.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.36.mlp.gate_proj.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.36.mlp.up_proj.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.36.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.36.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.36.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.36.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.36.self_attn.o_proj.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.36.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.36.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.37.input_layernorm.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.37.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.37.mlp.gate_proj.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.37.mlp.up_proj.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.37.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.37.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.37.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.37.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.37.self_attn.o_proj.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.37.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.37.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.38.input_layernorm.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.38.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.38.mlp.gate_proj.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.38.mlp.up_proj.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.38.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.38.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.38.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.38.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.38.self_attn.o_proj.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.38.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.38.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.39.input_layernorm.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.39.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.39.mlp.gate_proj.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.39.mlp.up_proj.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.39.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.39.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.39.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.39.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.39.self_attn.o_proj.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.39.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.39.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.4.input_layernorm.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.4.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.4.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.4.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.4.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.4.post_feedforward_layernorm.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.4.pre_feedforward_layernorm.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.4.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.4.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.4.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.4.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.40.input_layernorm.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.40.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.40.mlp.gate_proj.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.40.mlp.up_proj.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.40.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.40.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.40.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.40.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.40.self_attn.o_proj.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.40.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.40.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.41.input_layernorm.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.41.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.41.mlp.gate_proj.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.41.mlp.up_proj.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.41.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.41.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.41.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.41.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.41.self_attn.o_proj.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.41.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.41.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.5.input_layernorm.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.5.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.5.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.5.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.5.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.5.post_feedforward_layernorm.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.5.pre_feedforward_layernorm.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.5.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.5.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.5.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.5.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.6.input_layernorm.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.6.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.6.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.6.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.6.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.6.post_feedforward_layernorm.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.6.pre_feedforward_layernorm.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.6.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.6.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.6.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.6.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.7.input_layernorm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.7.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.7.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.7.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.7.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.7.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.7.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.7.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.7.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.7.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.7.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.8.input_layernorm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.8.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.8.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.8.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.8.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.8.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.8.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.8.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.8.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.8.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.8.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.9.input_layernorm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.9.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.9.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.9.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.9.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.9.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.9.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.9.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.9.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.9.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.9.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.norm.weight": "model-00004-of-00004.safetensors"
|
||||
}
|
||||
}
|
||||
34
special_tokens_map.json
Normal file
34
special_tokens_map.json
Normal file
@@ -0,0 +1,34 @@
|
||||
{
|
||||
"additional_special_tokens": [
|
||||
"<start_of_turn>",
|
||||
"<end_of_turn>"
|
||||
],
|
||||
"bos_token": {
|
||||
"content": "<bos>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false
|
||||
},
|
||||
"eos_token": {
|
||||
"content": "<eos>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false
|
||||
},
|
||||
"pad_token": {
|
||||
"content": "<pad>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false
|
||||
},
|
||||
"unk_token": {
|
||||
"content": "<unk>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false
|
||||
}
|
||||
}
|
||||
3
tokenizer.json
Normal file
3
tokenizer.json
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:7da53ca29fb16f6b2489482fc0bc6a394162cdab14d12764a1755ebc583fea79
|
||||
size 17518525
|
||||
3
tokenizer.model
Normal file
3
tokenizer.model
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:61a7b147390c64585d6c3543dd6fc636906c9af3865a5548f27f31aee1d4c8e2
|
||||
size 4241003
|
||||
1759
tokenizer_config.json
Normal file
1759
tokenizer_config.json
Normal file
File diff suppressed because it is too large
Load Diff
18
train_results.json
Normal file
18
train_results.json
Normal file
@@ -0,0 +1,18 @@
|
||||
{
|
||||
"before_init_mem_cpu": 3802071040,
|
||||
"before_init_mem_gpu": 22016,
|
||||
"epoch": 0.9807355516637478,
|
||||
"init_mem_cpu_alloc_delta": 364544,
|
||||
"init_mem_cpu_peaked_delta": 0,
|
||||
"init_mem_gpu_alloc_delta": 0,
|
||||
"init_mem_gpu_peaked_delta": 0,
|
||||
"total_flos": 39867492466688.0,
|
||||
"train_loss": 3.085822834287371,
|
||||
"train_mem_cpu_alloc_delta": 5213659136,
|
||||
"train_mem_cpu_peaked_delta": 22737326080,
|
||||
"train_mem_gpu_alloc_delta": 16267848704,
|
||||
"train_mem_gpu_peaked_delta": 36029468160,
|
||||
"train_runtime": 1628.7465,
|
||||
"train_samples_per_second": 2.805,
|
||||
"train_steps_per_second": 0.021
|
||||
}
|
||||
37
trainer_log.jsonl
Normal file
37
trainer_log.jsonl
Normal file
@@ -0,0 +1,37 @@
|
||||
{"current_steps": 1, "total_steps": 35, "loss": 6.8307, "accuracy": 0.5, "learning_rate": 8e-08, "epoch": 0.028021015761821366, "percentage": 2.86, "elapsed_time": "0:00:53", "remaining_time": "0:30:04", "throughput": "0.00", "total_tokens": 0}
|
||||
{"current_steps": 2, "total_steps": 35, "loss": 7.3627, "accuracy": 0.5, "learning_rate": 2.6e-07, "epoch": 0.05604203152364273, "percentage": 5.71, "elapsed_time": "0:01:44", "remaining_time": "0:28:51", "throughput": "0.00", "total_tokens": 0}
|
||||
{"current_steps": 3, "total_steps": 35, "loss": 7.025, "accuracy": 0.6875, "learning_rate": 4.4e-07, "epoch": 0.0840630472854641, "percentage": 8.57, "elapsed_time": "0:02:25", "remaining_time": "0:25:54", "throughput": "0.00", "total_tokens": 0}
|
||||
{"current_steps": 4, "total_steps": 35, "loss": 6.4674, "accuracy": 0.3125, "learning_rate": 6.2e-07, "epoch": 0.11208406304728546, "percentage": 11.43, "elapsed_time": "0:03:05", "remaining_time": "0:23:59", "throughput": "0.00", "total_tokens": 0}
|
||||
{"current_steps": 5, "total_steps": 35, "loss": 7.1364, "accuracy": 0.625, "learning_rate": 7.981529564210822e-07, "epoch": 0.14010507880910683, "percentage": 14.29, "elapsed_time": "0:03:49", "remaining_time": "0:22:59", "throughput": "0.00", "total_tokens": 0}
|
||||
{"current_steps": 6, "total_steps": 35, "loss": 5.6445, "accuracy": 0.6875, "learning_rate": 7.926307788508979e-07, "epoch": 0.1681260945709282, "percentage": 17.14, "elapsed_time": "0:04:31", "remaining_time": "0:21:53", "throughput": "0.00", "total_tokens": 0}
|
||||
{"current_steps": 7, "total_steps": 35, "loss": 6.5965, "accuracy": 0.375, "learning_rate": 7.834901323040175e-07, "epoch": 0.19614711033274956, "percentage": 20.0, "elapsed_time": "0:05:14", "remaining_time": "0:20:56", "throughput": "0.00", "total_tokens": 0}
|
||||
{"current_steps": 8, "total_steps": 35, "loss": 6.3454, "accuracy": 0.6875, "learning_rate": 7.70824812183283e-07, "epoch": 0.22416812609457093, "percentage": 22.86, "elapsed_time": "0:05:55", "remaining_time": "0:19:58", "throughput": "0.00", "total_tokens": 0}
|
||||
{"current_steps": 9, "total_steps": 35, "loss": 5.2256, "accuracy": 0.5, "learning_rate": 7.547647818120495e-07, "epoch": 0.2521891418563923, "percentage": 25.71, "elapsed_time": "0:06:40", "remaining_time": "0:19:16", "throughput": "0.00", "total_tokens": 0}
|
||||
{"current_steps": 10, "total_steps": 35, "loss": 4.8325, "accuracy": 0.5625, "learning_rate": 7.354748388346194e-07, "epoch": 0.28021015761821366, "percentage": 28.57, "elapsed_time": "0:07:19", "remaining_time": "0:18:18", "throughput": "0.00", "total_tokens": 0}
|
||||
{"current_steps": 11, "total_steps": 35, "loss": 4.8686, "accuracy": 0.875, "learning_rate": 7.131529241694047e-07, "epoch": 0.30823117338003503, "percentage": 31.43, "elapsed_time": "0:07:58", "remaining_time": "0:17:24", "throughput": "0.00", "total_tokens": 0}
|
||||
{"current_steps": 12, "total_steps": 35, "loss": 4.1241, "accuracy": 0.625, "learning_rate": 6.880280908672471e-07, "epoch": 0.3362521891418564, "percentage": 34.29, "elapsed_time": "0:08:42", "remaining_time": "0:16:40", "throughput": "0.00", "total_tokens": 0}
|
||||
{"current_steps": 13, "total_steps": 35, "loss": 3.8904, "accuracy": 0.75, "learning_rate": 6.603581537171586e-07, "epoch": 0.36427320490367776, "percentage": 37.14, "elapsed_time": "0:09:20", "remaining_time": "0:15:48", "throughput": "0.00", "total_tokens": 0}
|
||||
{"current_steps": 14, "total_steps": 35, "loss": 2.6976, "accuracy": 0.75, "learning_rate": 6.304270437177064e-07, "epoch": 0.3922942206654991, "percentage": 40.0, "elapsed_time": "0:10:04", "remaining_time": "0:15:07", "throughput": "0.00", "total_tokens": 0}
|
||||
{"current_steps": 15, "total_steps": 35, "loss": 1.2788, "accuracy": 0.9375, "learning_rate": 5.985418945607484e-07, "epoch": 0.4203152364273205, "percentage": 42.86, "elapsed_time": "0:10:51", "remaining_time": "0:14:28", "throughput": "0.00", "total_tokens": 0}
|
||||
{"current_steps": 16, "total_steps": 35, "loss": 1.4116, "accuracy": 0.875, "learning_rate": 5.650298910241353e-07, "epoch": 0.44833625218914186, "percentage": 45.71, "elapsed_time": "0:11:32", "remaining_time": "0:13:42", "throughput": "0.00", "total_tokens": 0}
|
||||
{"current_steps": 17, "total_steps": 35, "loss": 1.6013, "accuracy": 0.9375, "learning_rate": 5.302349116131393e-07, "epoch": 0.4763572679509632, "percentage": 48.57, "elapsed_time": "0:12:09", "remaining_time": "0:12:52", "throughput": "0.00", "total_tokens": 0}
|
||||
{"current_steps": 18, "total_steps": 35, "loss": 1.8319, "accuracy": 0.875, "learning_rate": 4.945139999016476e-07, "epoch": 0.5043782837127846, "percentage": 51.43, "elapsed_time": "0:12:43", "remaining_time": "0:12:01", "throughput": "0.00", "total_tokens": 0}
|
||||
{"current_steps": 19, "total_steps": 35, "loss": 1.7772, "accuracy": 0.8125, "learning_rate": 4.5823370078193663e-07, "epoch": 0.532399299474606, "percentage": 54.29, "elapsed_time": "0:13:23", "remaining_time": "0:11:16", "throughput": "0.00", "total_tokens": 0}
|
||||
{"current_steps": 20, "total_steps": 35, "loss": 1.6103, "accuracy": 0.8125, "learning_rate": 4.217662992180634e-07, "epoch": 0.5604203152364273, "percentage": 57.14, "elapsed_time": "0:14:03", "remaining_time": "0:10:32", "throughput": "0.00", "total_tokens": 0}
|
||||
{"current_steps": 21, "total_steps": 35, "loss": 1.4708, "accuracy": 0.9375, "learning_rate": 3.8548600009835237e-07, "epoch": 0.5884413309982487, "percentage": 60.0, "elapsed_time": "0:14:44", "remaining_time": "0:09:49", "throughput": "0.00", "total_tokens": 0}
|
||||
{"current_steps": 22, "total_steps": 35, "loss": 1.1655, "accuracy": 1.0, "learning_rate": 3.4976508838686066e-07, "epoch": 0.6164623467600701, "percentage": 62.86, "elapsed_time": "0:15:32", "remaining_time": "0:09:11", "throughput": "0.00", "total_tokens": 0}
|
||||
{"current_steps": 23, "total_steps": 35, "loss": 1.2242, "accuracy": 0.9375, "learning_rate": 3.149701089758648e-07, "epoch": 0.6444833625218914, "percentage": 65.71, "elapsed_time": "0:16:13", "remaining_time": "0:08:28", "throughput": "0.00", "total_tokens": 0}
|
||||
{"current_steps": 24, "total_steps": 35, "loss": 1.4752, "accuracy": 0.9375, "learning_rate": 2.8145810543925163e-07, "epoch": 0.6725043782837128, "percentage": 68.57, "elapsed_time": "0:16:57", "remaining_time": "0:07:46", "throughput": "0.00", "total_tokens": 0}
|
||||
{"current_steps": 25, "total_steps": 35, "loss": 1.4732, "accuracy": 1.0, "learning_rate": 2.495729562822935e-07, "epoch": 0.7005253940455342, "percentage": 71.43, "elapsed_time": "0:17:39", "remaining_time": "0:07:03", "throughput": "0.00", "total_tokens": 0}
|
||||
{"current_steps": 26, "total_steps": 35, "loss": 1.2593, "accuracy": 0.8125, "learning_rate": 2.196418462828415e-07, "epoch": 0.7285464098073555, "percentage": 74.29, "elapsed_time": "0:18:24", "remaining_time": "0:06:22", "throughput": "0.00", "total_tokens": 0}
|
||||
{"current_steps": 27, "total_steps": 35, "loss": 1.1459, "accuracy": 0.8125, "learning_rate": 1.9197190913275294e-07, "epoch": 0.7565674255691769, "percentage": 77.14, "elapsed_time": "0:19:07", "remaining_time": "0:05:40", "throughput": "0.00", "total_tokens": 0}
|
||||
{"current_steps": 28, "total_steps": 35, "loss": 1.3002, "accuracy": 0.9375, "learning_rate": 1.6684707583059529e-07, "epoch": 0.7845884413309983, "percentage": 80.0, "elapsed_time": "0:19:48", "remaining_time": "0:04:57", "throughput": "0.00", "total_tokens": 0}
|
||||
{"current_steps": 29, "total_steps": 35, "loss": 0.8862, "accuracy": 0.9375, "learning_rate": 1.4452516116538054e-07, "epoch": 0.8126094570928196, "percentage": 82.86, "elapsed_time": "0:20:27", "remaining_time": "0:04:14", "throughput": "0.00", "total_tokens": 0}
|
||||
{"current_steps": 30, "total_steps": 35, "loss": 1.3672, "accuracy": 1.0, "learning_rate": 1.2523521818795044e-07, "epoch": 0.840630472854641, "percentage": 85.71, "elapsed_time": "0:21:14", "remaining_time": "0:03:32", "throughput": "0.00", "total_tokens": 0}
|
||||
{"current_steps": 31, "total_steps": 35, "loss": 1.3672, "accuracy": 0.875, "learning_rate": 1.0917518781671699e-07, "epoch": 0.8686514886164624, "percentage": 88.57, "elapsed_time": "0:21:58", "remaining_time": "0:02:50", "throughput": "0.00", "total_tokens": 0}
|
||||
{"current_steps": 32, "total_steps": 35, "loss": 1.6485, "accuracy": 0.9375, "learning_rate": 9.650986769598242e-08, "epoch": 0.8966725043782837, "percentage": 91.43, "elapsed_time": "0:22:44", "remaining_time": "0:02:07", "throughput": "0.00", "total_tokens": 0}
|
||||
{"current_steps": 33, "total_steps": 35, "loss": 1.3242, "accuracy": 0.9375, "learning_rate": 8.736922114910199e-08, "epoch": 0.9246935201401051, "percentage": 94.29, "elapsed_time": "0:23:28", "remaining_time": "0:01:25", "throughput": "0.00", "total_tokens": 0}
|
||||
{"current_steps": 34, "total_steps": 35, "loss": 0.8932, "accuracy": 0.9375, "learning_rate": 8.184704357891779e-08, "epoch": 0.9527145359019265, "percentage": 97.14, "elapsed_time": "0:24:10", "remaining_time": "0:00:42", "throughput": "0.00", "total_tokens": 0}
|
||||
{"current_steps": 35, "total_steps": 35, "loss": 1.4444, "accuracy": 0.9375, "learning_rate": 8e-08, "epoch": 0.9807355516637478, "percentage": 100.0, "elapsed_time": "0:24:50", "remaining_time": "0:00:00", "throughput": "0.00", "total_tokens": 0}
|
||||
{"current_steps": 35, "total_steps": 35, "eval_loss": 1.0162526369094849, "epoch": 0.9807355516637478, "percentage": 100.0, "elapsed_time": "0:26:02", "remaining_time": "0:00:00", "throughput": "0.00", "total_tokens": 0}
|
||||
{"current_steps": 35, "total_steps": 35, "epoch": 0.9807355516637478, "percentage": 100.0, "elapsed_time": "0:27:09", "remaining_time": "0:00:00", "throughput": "0.00", "total_tokens": 0}
|
||||
629
trainer_state.json
Normal file
629
trainer_state.json
Normal file
@@ -0,0 +1,629 @@
|
||||
{
|
||||
"best_metric": null,
|
||||
"best_model_checkpoint": null,
|
||||
"epoch": 0.9807355516637478,
|
||||
"eval_steps": 18,
|
||||
"global_step": 35,
|
||||
"is_hyper_param_search": false,
|
||||
"is_local_process_zero": true,
|
||||
"is_world_process_zero": true,
|
||||
"log_history": [
|
||||
{
|
||||
"epoch": 0.028021015761821366,
|
||||
"grad_norm": 97.3614384660342,
|
||||
"learning_rate": 8e-08,
|
||||
"logits/chosen": -7.731139183044434,
|
||||
"logits/rejected": -8.25313663482666,
|
||||
"logps/chosen": -1.6605920791625977,
|
||||
"logps/rejected": -1.683160424232483,
|
||||
"loss": 6.8307,
|
||||
"rewards/accuracies": 0.5,
|
||||
"rewards/chosen": -16.60592269897461,
|
||||
"rewards/margins": 0.22568130493164062,
|
||||
"rewards/rejected": -16.83160400390625,
|
||||
"sft_loss": 0.04547927528619766,
|
||||
"step": 1
|
||||
},
|
||||
{
|
||||
"epoch": 0.05604203152364273,
|
||||
"grad_norm": 121.91139131956491,
|
||||
"learning_rate": 2.6e-07,
|
||||
"logits/chosen": -7.893815040588379,
|
||||
"logits/rejected": -8.019620895385742,
|
||||
"logps/chosen": -1.952430248260498,
|
||||
"logps/rejected": -1.8113142251968384,
|
||||
"loss": 7.3627,
|
||||
"rewards/accuracies": 0.5,
|
||||
"rewards/chosen": -19.52429962158203,
|
||||
"rewards/margins": -1.4111591577529907,
|
||||
"rewards/rejected": -18.113142013549805,
|
||||
"sft_loss": 0.00891563668847084,
|
||||
"step": 2
|
||||
},
|
||||
{
|
||||
"epoch": 0.0840630472854641,
|
||||
"grad_norm": 115.99875317435591,
|
||||
"learning_rate": 4.4e-07,
|
||||
"logits/chosen": -9.035008430480957,
|
||||
"logits/rejected": -8.950678825378418,
|
||||
"logps/chosen": -1.4869132041931152,
|
||||
"logps/rejected": -1.50464928150177,
|
||||
"loss": 7.025,
|
||||
"rewards/accuracies": 0.6875,
|
||||
"rewards/chosen": -14.869132995605469,
|
||||
"rewards/margins": 0.17736005783081055,
|
||||
"rewards/rejected": -15.046493530273438,
|
||||
"sft_loss": 0.010355046950280666,
|
||||
"step": 3
|
||||
},
|
||||
{
|
||||
"epoch": 0.11208406304728546,
|
||||
"grad_norm": 143.98542912632848,
|
||||
"learning_rate": 6.2e-07,
|
||||
"logits/chosen": -9.924211502075195,
|
||||
"logits/rejected": -9.45657730102539,
|
||||
"logps/chosen": -1.4292300939559937,
|
||||
"logps/rejected": -1.2033114433288574,
|
||||
"loss": 6.4674,
|
||||
"rewards/accuracies": 0.3125,
|
||||
"rewards/chosen": -14.292301177978516,
|
||||
"rewards/margins": -2.2591874599456787,
|
||||
"rewards/rejected": -12.033114433288574,
|
||||
"sft_loss": 0.006642716005444527,
|
||||
"step": 4
|
||||
},
|
||||
{
|
||||
"epoch": 0.14010507880910683,
|
||||
"grad_norm": 105.0416363604681,
|
||||
"learning_rate": 7.981529564210822e-07,
|
||||
"logits/chosen": -8.538932800292969,
|
||||
"logits/rejected": -8.046061515808105,
|
||||
"logps/chosen": -1.5071882009506226,
|
||||
"logps/rejected": -1.675721526145935,
|
||||
"loss": 7.1364,
|
||||
"rewards/accuracies": 0.625,
|
||||
"rewards/chosen": -15.071882247924805,
|
||||
"rewards/margins": 1.6853327751159668,
|
||||
"rewards/rejected": -16.757217407226562,
|
||||
"sft_loss": 0.01169000007212162,
|
||||
"step": 5
|
||||
},
|
||||
{
|
||||
"epoch": 0.1681260945709282,
|
||||
"grad_norm": 136.46572113440772,
|
||||
"learning_rate": 7.926307788508979e-07,
|
||||
"logits/chosen": -8.856929779052734,
|
||||
"logits/rejected": -9.344861030578613,
|
||||
"logps/chosen": -1.6353546380996704,
|
||||
"logps/rejected": -1.7302504777908325,
|
||||
"loss": 5.6445,
|
||||
"rewards/accuracies": 0.6875,
|
||||
"rewards/chosen": -16.353546142578125,
|
||||
"rewards/margins": 0.9489572644233704,
|
||||
"rewards/rejected": -17.30250358581543,
|
||||
"sft_loss": 0.03519538417458534,
|
||||
"step": 6
|
||||
},
|
||||
{
|
||||
"epoch": 0.19614711033274956,
|
||||
"grad_norm": 125.56078536529542,
|
||||
"learning_rate": 7.834901323040175e-07,
|
||||
"logits/chosen": -7.622992515563965,
|
||||
"logits/rejected": -7.627020835876465,
|
||||
"logps/chosen": -2.047703981399536,
|
||||
"logps/rejected": -1.6984105110168457,
|
||||
"loss": 6.5965,
|
||||
"rewards/accuracies": 0.375,
|
||||
"rewards/chosen": -20.477039337158203,
|
||||
"rewards/margins": -3.4929349422454834,
|
||||
"rewards/rejected": -16.98410415649414,
|
||||
"sft_loss": 0.006214356515556574,
|
||||
"step": 7
|
||||
},
|
||||
{
|
||||
"epoch": 0.22416812609457093,
|
||||
"grad_norm": 135.07237608120852,
|
||||
"learning_rate": 7.70824812183283e-07,
|
||||
"logits/chosen": -8.93583869934082,
|
||||
"logits/rejected": -9.021809577941895,
|
||||
"logps/chosen": -1.3730394840240479,
|
||||
"logps/rejected": -1.4458472728729248,
|
||||
"loss": 6.3454,
|
||||
"rewards/accuracies": 0.6875,
|
||||
"rewards/chosen": -13.730398178100586,
|
||||
"rewards/margins": 0.7280769944190979,
|
||||
"rewards/rejected": -14.458473205566406,
|
||||
"sft_loss": 0.03004990890622139,
|
||||
"step": 8
|
||||
},
|
||||
{
|
||||
"epoch": 0.2521891418563923,
|
||||
"grad_norm": 239.98627324152338,
|
||||
"learning_rate": 7.547647818120495e-07,
|
||||
"logits/chosen": -9.406291961669922,
|
||||
"logits/rejected": -9.9324951171875,
|
||||
"logps/chosen": -1.5991909503936768,
|
||||
"logps/rejected": -1.6271830797195435,
|
||||
"loss": 5.2256,
|
||||
"rewards/accuracies": 0.5,
|
||||
"rewards/chosen": -15.991909980773926,
|
||||
"rewards/margins": 0.2799214720726013,
|
||||
"rewards/rejected": -16.271831512451172,
|
||||
"sft_loss": 0.03231532499194145,
|
||||
"step": 9
|
||||
},
|
||||
{
|
||||
"epoch": 0.28021015761821366,
|
||||
"grad_norm": 234.53437032749468,
|
||||
"learning_rate": 7.354748388346194e-07,
|
||||
"logits/chosen": -7.813473701477051,
|
||||
"logits/rejected": -9.137899398803711,
|
||||
"logps/chosen": -1.8943036794662476,
|
||||
"logps/rejected": -1.9154584407806396,
|
||||
"loss": 4.8325,
|
||||
"rewards/accuracies": 0.5625,
|
||||
"rewards/chosen": -18.943037033081055,
|
||||
"rewards/margins": 0.2115485668182373,
|
||||
"rewards/rejected": -19.154584884643555,
|
||||
"sft_loss": 0.008024048060178757,
|
||||
"step": 10
|
||||
},
|
||||
{
|
||||
"epoch": 0.30823117338003503,
|
||||
"grad_norm": 287.5499688149926,
|
||||
"learning_rate": 7.131529241694047e-07,
|
||||
"logits/chosen": -10.093656539916992,
|
||||
"logits/rejected": -10.823583602905273,
|
||||
"logps/chosen": -1.676328182220459,
|
||||
"logps/rejected": -2.193615198135376,
|
||||
"loss": 4.8686,
|
||||
"rewards/accuracies": 0.875,
|
||||
"rewards/chosen": -16.763280868530273,
|
||||
"rewards/margins": 5.17287015914917,
|
||||
"rewards/rejected": -21.9361515045166,
|
||||
"sft_loss": 0.018158258870244026,
|
||||
"step": 11
|
||||
},
|
||||
{
|
||||
"epoch": 0.3362521891418564,
|
||||
"grad_norm": 279.35544138821984,
|
||||
"learning_rate": 6.880280908672471e-07,
|
||||
"logits/chosen": -7.598231792449951,
|
||||
"logits/rejected": -8.863749504089355,
|
||||
"logps/chosen": -1.8558087348937988,
|
||||
"logps/rejected": -2.048665761947632,
|
||||
"loss": 4.1241,
|
||||
"rewards/accuracies": 0.625,
|
||||
"rewards/chosen": -18.558086395263672,
|
||||
"rewards/margins": 1.9285707473754883,
|
||||
"rewards/rejected": -20.486658096313477,
|
||||
"sft_loss": 0.003531986614689231,
|
||||
"step": 12
|
||||
},
|
||||
{
|
||||
"epoch": 0.36427320490367776,
|
||||
"grad_norm": 308.20678092603185,
|
||||
"learning_rate": 6.603581537171586e-07,
|
||||
"logits/chosen": -8.397397994995117,
|
||||
"logits/rejected": -10.13599681854248,
|
||||
"logps/chosen": -1.745999813079834,
|
||||
"logps/rejected": -1.8406281471252441,
|
||||
"loss": 3.8904,
|
||||
"rewards/accuracies": 0.75,
|
||||
"rewards/chosen": -17.459999084472656,
|
||||
"rewards/margins": 0.9462810754776001,
|
||||
"rewards/rejected": -18.406280517578125,
|
||||
"sft_loss": 0.010710272938013077,
|
||||
"step": 13
|
||||
},
|
||||
{
|
||||
"epoch": 0.3922942206654991,
|
||||
"grad_norm": 88.53513750352447,
|
||||
"learning_rate": 6.304270437177064e-07,
|
||||
"logits/chosen": -11.391769409179688,
|
||||
"logits/rejected": -13.656466484069824,
|
||||
"logps/chosen": -1.820195198059082,
|
||||
"logps/rejected": -2.2572662830352783,
|
||||
"loss": 2.6976,
|
||||
"rewards/accuracies": 0.75,
|
||||
"rewards/chosen": -18.201950073242188,
|
||||
"rewards/margins": 4.370712757110596,
|
||||
"rewards/rejected": -22.572664260864258,
|
||||
"sft_loss": 0.006363618653267622,
|
||||
"step": 14
|
||||
},
|
||||
{
|
||||
"epoch": 0.4203152364273205,
|
||||
"grad_norm": 82.22128437782617,
|
||||
"learning_rate": 5.985418945607484e-07,
|
||||
"logits/chosen": -12.035834312438965,
|
||||
"logits/rejected": -14.527205467224121,
|
||||
"logps/chosen": -2.3111374378204346,
|
||||
"logps/rejected": -3.1396684646606445,
|
||||
"loss": 1.2788,
|
||||
"rewards/accuracies": 0.9375,
|
||||
"rewards/chosen": -23.111371994018555,
|
||||
"rewards/margins": 8.285309791564941,
|
||||
"rewards/rejected": -31.396682739257812,
|
||||
"sft_loss": 0.007131902035325766,
|
||||
"step": 15
|
||||
},
|
||||
{
|
||||
"epoch": 0.44833625218914186,
|
||||
"grad_norm": 120.40204277011782,
|
||||
"learning_rate": 5.650298910241353e-07,
|
||||
"logits/chosen": -12.784805297851562,
|
||||
"logits/rejected": -15.177325248718262,
|
||||
"logps/chosen": -2.0324227809906006,
|
||||
"logps/rejected": -3.119992971420288,
|
||||
"loss": 1.4116,
|
||||
"rewards/accuracies": 0.875,
|
||||
"rewards/chosen": -20.324228286743164,
|
||||
"rewards/margins": 10.875699996948242,
|
||||
"rewards/rejected": -31.199928283691406,
|
||||
"sft_loss": 0.005817623808979988,
|
||||
"step": 16
|
||||
},
|
||||
{
|
||||
"epoch": 0.4763572679509632,
|
||||
"grad_norm": 97.73770625062221,
|
||||
"learning_rate": 5.302349116131393e-07,
|
||||
"logits/chosen": -15.652244567871094,
|
||||
"logits/rejected": -17.80880355834961,
|
||||
"logps/chosen": -1.9218964576721191,
|
||||
"logps/rejected": -2.738666296005249,
|
||||
"loss": 1.6013,
|
||||
"rewards/accuracies": 0.9375,
|
||||
"rewards/chosen": -19.218965530395508,
|
||||
"rewards/margins": 8.167696952819824,
|
||||
"rewards/rejected": -27.386659622192383,
|
||||
"sft_loss": 0.004993550945073366,
|
||||
"step": 17
|
||||
},
|
||||
{
|
||||
"epoch": 0.5043782837127846,
|
||||
"grad_norm": 189.96208267597956,
|
||||
"learning_rate": 4.945139999016476e-07,
|
||||
"logits/chosen": -15.17531681060791,
|
||||
"logits/rejected": -15.677058219909668,
|
||||
"logps/chosen": -1.9880082607269287,
|
||||
"logps/rejected": -3.052130699157715,
|
||||
"loss": 1.8319,
|
||||
"rewards/accuracies": 0.875,
|
||||
"rewards/chosen": -19.880081176757812,
|
||||
"rewards/margins": 10.641225814819336,
|
||||
"rewards/rejected": -30.52130889892578,
|
||||
"sft_loss": 0.008743491023778915,
|
||||
"step": 18
|
||||
},
|
||||
{
|
||||
"epoch": 0.532399299474606,
|
||||
"grad_norm": 114.52318633727846,
|
||||
"learning_rate": 4.5823370078193663e-07,
|
||||
"logits/chosen": -10.547327995300293,
|
||||
"logits/rejected": -14.194029808044434,
|
||||
"logps/chosen": -2.2875614166259766,
|
||||
"logps/rejected": -3.8111658096313477,
|
||||
"loss": 1.7772,
|
||||
"rewards/accuracies": 0.8125,
|
||||
"rewards/chosen": -22.875612258911133,
|
||||
"rewards/margins": 15.236043930053711,
|
||||
"rewards/rejected": -38.111656188964844,
|
||||
"sft_loss": 0.01470925658941269,
|
||||
"step": 19
|
||||
},
|
||||
{
|
||||
"epoch": 0.5604203152364273,
|
||||
"grad_norm": 82.97272038448429,
|
||||
"learning_rate": 4.217662992180634e-07,
|
||||
"logits/chosen": -10.6708345413208,
|
||||
"logits/rejected": -15.503955841064453,
|
||||
"logps/chosen": -2.2511441707611084,
|
||||
"logps/rejected": -3.763054132461548,
|
||||
"loss": 1.6103,
|
||||
"rewards/accuracies": 0.8125,
|
||||
"rewards/chosen": -22.511442184448242,
|
||||
"rewards/margins": 15.119099617004395,
|
||||
"rewards/rejected": -37.63053894042969,
|
||||
"sft_loss": 0.012447498738765717,
|
||||
"step": 20
|
||||
},
|
||||
{
|
||||
"epoch": 0.5884413309982487,
|
||||
"grad_norm": 93.13289791788793,
|
||||
"learning_rate": 3.8548600009835237e-07,
|
||||
"logits/chosen": -11.672554016113281,
|
||||
"logits/rejected": -16.79704475402832,
|
||||
"logps/chosen": -2.7983806133270264,
|
||||
"logps/rejected": -4.703005790710449,
|
||||
"loss": 1.4708,
|
||||
"rewards/accuracies": 0.9375,
|
||||
"rewards/chosen": -27.983802795410156,
|
||||
"rewards/margins": 19.046255111694336,
|
||||
"rewards/rejected": -47.030059814453125,
|
||||
"sft_loss": 0.0047310409136116505,
|
||||
"step": 21
|
||||
},
|
||||
{
|
||||
"epoch": 0.6164623467600701,
|
||||
"grad_norm": 97.54389148924957,
|
||||
"learning_rate": 3.4976508838686066e-07,
|
||||
"logits/chosen": -16.980070114135742,
|
||||
"logits/rejected": -18.32730484008789,
|
||||
"logps/chosen": -2.150972843170166,
|
||||
"logps/rejected": -3.424234390258789,
|
||||
"loss": 1.1655,
|
||||
"rewards/accuracies": 1.0,
|
||||
"rewards/chosen": -21.509729385375977,
|
||||
"rewards/margins": 12.732614517211914,
|
||||
"rewards/rejected": -34.242340087890625,
|
||||
"sft_loss": 0.007569438312202692,
|
||||
"step": 22
|
||||
},
|
||||
{
|
||||
"epoch": 0.6444833625218914,
|
||||
"grad_norm": 73.41731467567557,
|
||||
"learning_rate": 3.149701089758648e-07,
|
||||
"logits/chosen": -11.438061714172363,
|
||||
"logits/rejected": -14.675212860107422,
|
||||
"logps/chosen": -2.3481569290161133,
|
||||
"logps/rejected": -4.2203450202941895,
|
||||
"loss": 1.2242,
|
||||
"rewards/accuracies": 0.9375,
|
||||
"rewards/chosen": -23.481565475463867,
|
||||
"rewards/margins": 18.721879959106445,
|
||||
"rewards/rejected": -42.20344543457031,
|
||||
"sft_loss": 0.010319937951862812,
|
||||
"step": 23
|
||||
},
|
||||
{
|
||||
"epoch": 0.6725043782837128,
|
||||
"grad_norm": 96.48356219878691,
|
||||
"learning_rate": 2.8145810543925163e-07,
|
||||
"logits/chosen": -11.793488502502441,
|
||||
"logits/rejected": -15.576847076416016,
|
||||
"logps/chosen": -2.3053696155548096,
|
||||
"logps/rejected": -4.343653202056885,
|
||||
"loss": 1.4752,
|
||||
"rewards/accuracies": 0.9375,
|
||||
"rewards/chosen": -23.053693771362305,
|
||||
"rewards/margins": 20.382837295532227,
|
||||
"rewards/rejected": -43.4365348815918,
|
||||
"sft_loss": 0.024834871292114258,
|
||||
"step": 24
|
||||
},
|
||||
{
|
||||
"epoch": 0.7005253940455342,
|
||||
"grad_norm": 122.92321660024119,
|
||||
"learning_rate": 2.495729562822935e-07,
|
||||
"logits/chosen": -16.064531326293945,
|
||||
"logits/rejected": -17.884010314941406,
|
||||
"logps/chosen": -2.647796869277954,
|
||||
"logps/rejected": -3.9600863456726074,
|
||||
"loss": 1.4732,
|
||||
"rewards/accuracies": 1.0,
|
||||
"rewards/chosen": -26.477968215942383,
|
||||
"rewards/margins": 13.12289810180664,
|
||||
"rewards/rejected": -39.60086441040039,
|
||||
"sft_loss": 0.021936513483524323,
|
||||
"step": 25
|
||||
},
|
||||
{
|
||||
"epoch": 0.7285464098073555,
|
||||
"grad_norm": 80.26268266585235,
|
||||
"learning_rate": 2.196418462828415e-07,
|
||||
"logits/chosen": -12.343572616577148,
|
||||
"logits/rejected": -15.574173927307129,
|
||||
"logps/chosen": -2.2388336658477783,
|
||||
"logps/rejected": -4.106793403625488,
|
||||
"loss": 1.2593,
|
||||
"rewards/accuracies": 0.8125,
|
||||
"rewards/chosen": -22.388338088989258,
|
||||
"rewards/margins": 18.679595947265625,
|
||||
"rewards/rejected": -41.06793212890625,
|
||||
"sft_loss": 0.013025043532252312,
|
||||
"step": 26
|
||||
},
|
||||
{
|
||||
"epoch": 0.7565674255691769,
|
||||
"grad_norm": 66.28989889413502,
|
||||
"learning_rate": 1.9197190913275294e-07,
|
||||
"logits/chosen": -12.330286026000977,
|
||||
"logits/rejected": -15.901168823242188,
|
||||
"logps/chosen": -2.5048580169677734,
|
||||
"logps/rejected": -4.5386857986450195,
|
||||
"loss": 1.1459,
|
||||
"rewards/accuracies": 0.8125,
|
||||
"rewards/chosen": -25.048580169677734,
|
||||
"rewards/margins": 20.338275909423828,
|
||||
"rewards/rejected": -45.38685607910156,
|
||||
"sft_loss": 0.006859698798507452,
|
||||
"step": 27
|
||||
},
|
||||
{
|
||||
"epoch": 0.7845884413309983,
|
||||
"grad_norm": 81.14927404545568,
|
||||
"learning_rate": 1.6684707583059529e-07,
|
||||
"logits/chosen": -17.190406799316406,
|
||||
"logits/rejected": -19.05614471435547,
|
||||
"logps/chosen": -2.768648147583008,
|
||||
"logps/rejected": -4.382925033569336,
|
||||
"loss": 1.3002,
|
||||
"rewards/accuracies": 0.9375,
|
||||
"rewards/chosen": -27.68647575378418,
|
||||
"rewards/margins": 16.142770767211914,
|
||||
"rewards/rejected": -43.82925033569336,
|
||||
"sft_loss": 0.005794988479465246,
|
||||
"step": 28
|
||||
},
|
||||
{
|
||||
"epoch": 0.8126094570928196,
|
||||
"grad_norm": 70.98888001094448,
|
||||
"learning_rate": 1.4452516116538054e-07,
|
||||
"logits/chosen": -10.294852256774902,
|
||||
"logits/rejected": -15.053112030029297,
|
||||
"logps/chosen": -2.106778383255005,
|
||||
"logps/rejected": -4.66357421875,
|
||||
"loss": 0.8862,
|
||||
"rewards/accuracies": 0.9375,
|
||||
"rewards/chosen": -21.067781448364258,
|
||||
"rewards/margins": 25.567956924438477,
|
||||
"rewards/rejected": -46.6357421875,
|
||||
"sft_loss": 0.008147615939378738,
|
||||
"step": 29
|
||||
},
|
||||
{
|
||||
"epoch": 0.840630472854641,
|
||||
"grad_norm": 99.24795263935427,
|
||||
"learning_rate": 1.2523521818795044e-07,
|
||||
"logits/chosen": -9.799162864685059,
|
||||
"logits/rejected": -15.37686538696289,
|
||||
"logps/chosen": -2.270378828048706,
|
||||
"logps/rejected": -5.39860725402832,
|
||||
"loss": 1.3672,
|
||||
"rewards/accuracies": 1.0,
|
||||
"rewards/chosen": -22.70379066467285,
|
||||
"rewards/margins": 31.282283782958984,
|
||||
"rewards/rejected": -53.98607635498047,
|
||||
"sft_loss": 0.005661052651703358,
|
||||
"step": 30
|
||||
},
|
||||
{
|
||||
"epoch": 0.8686514886164624,
|
||||
"grad_norm": 105.42160708641147,
|
||||
"learning_rate": 1.0917518781671699e-07,
|
||||
"logits/chosen": -10.193641662597656,
|
||||
"logits/rejected": -13.275125503540039,
|
||||
"logps/chosen": -2.434311866760254,
|
||||
"logps/rejected": -3.9474875926971436,
|
||||
"loss": 1.3672,
|
||||
"rewards/accuracies": 0.875,
|
||||
"rewards/chosen": -24.343116760253906,
|
||||
"rewards/margins": 15.131752967834473,
|
||||
"rewards/rejected": -39.47487258911133,
|
||||
"sft_loss": 0.01922934129834175,
|
||||
"step": 31
|
||||
},
|
||||
{
|
||||
"epoch": 0.8966725043782837,
|
||||
"grad_norm": 75.46584751909548,
|
||||
"learning_rate": 9.650986769598242e-08,
|
||||
"logits/chosen": -9.355328559875488,
|
||||
"logits/rejected": -16.710058212280273,
|
||||
"logps/chosen": -2.3611741065979004,
|
||||
"logps/rejected": -5.737414360046387,
|
||||
"loss": 1.6485,
|
||||
"rewards/accuracies": 0.9375,
|
||||
"rewards/chosen": -23.61174201965332,
|
||||
"rewards/margins": 33.76239776611328,
|
||||
"rewards/rejected": -57.3741455078125,
|
||||
"sft_loss": 0.010620678775012493,
|
||||
"step": 32
|
||||
},
|
||||
{
|
||||
"epoch": 0.9246935201401051,
|
||||
"grad_norm": 84.00269490747272,
|
||||
"learning_rate": 8.736922114910199e-08,
|
||||
"logits/chosen": -10.082009315490723,
|
||||
"logits/rejected": -14.500289916992188,
|
||||
"logps/chosen": -2.227128267288208,
|
||||
"logps/rejected": -5.038478374481201,
|
||||
"loss": 1.3242,
|
||||
"rewards/accuracies": 0.9375,
|
||||
"rewards/chosen": -22.271284103393555,
|
||||
"rewards/margins": 28.113502502441406,
|
||||
"rewards/rejected": -50.384788513183594,
|
||||
"sft_loss": 0.009218152612447739,
|
||||
"step": 33
|
||||
},
|
||||
{
|
||||
"epoch": 0.9527145359019265,
|
||||
"grad_norm": 75.42419711071338,
|
||||
"learning_rate": 8.184704357891779e-08,
|
||||
"logits/chosen": -10.325528144836426,
|
||||
"logits/rejected": -16.278217315673828,
|
||||
"logps/chosen": -2.2617452144622803,
|
||||
"logps/rejected": -4.84274435043335,
|
||||
"loss": 0.8932,
|
||||
"rewards/accuracies": 0.9375,
|
||||
"rewards/chosen": -22.617454528808594,
|
||||
"rewards/margins": 25.809988021850586,
|
||||
"rewards/rejected": -48.42743682861328,
|
||||
"sft_loss": 0.014543527737259865,
|
||||
"step": 34
|
||||
},
|
||||
{
|
||||
"epoch": 0.9807355516637478,
|
||||
"grad_norm": 108.8748972501278,
|
||||
"learning_rate": 8e-08,
|
||||
"logits/chosen": -12.176219940185547,
|
||||
"logits/rejected": -16.480701446533203,
|
||||
"logps/chosen": -2.5212955474853516,
|
||||
"logps/rejected": -4.665997505187988,
|
||||
"loss": 1.4444,
|
||||
"rewards/accuracies": 0.9375,
|
||||
"rewards/chosen": -25.21295166015625,
|
||||
"rewards/margins": 21.447019577026367,
|
||||
"rewards/rejected": -46.65997314453125,
|
||||
"sft_loss": 0.04498640075325966,
|
||||
"step": 35
|
||||
},
|
||||
{
|
||||
"epoch": 0.9807355516637478,
|
||||
"eval_logits/chosen": -12.004097938537598,
|
||||
"eval_logits/rejected": -17.047502517700195,
|
||||
"eval_logps/chosen": -2.168222427368164,
|
||||
"eval_logps/rejected": -4.787535667419434,
|
||||
"eval_loss": 1.0162526369094849,
|
||||
"eval_rewards/accuracies": 0.9166666865348816,
|
||||
"eval_rewards/chosen": -21.682226181030273,
|
||||
"eval_rewards/margins": 26.193130493164062,
|
||||
"eval_rewards/rejected": -47.875362396240234,
|
||||
"eval_runtime": 9.3123,
|
||||
"eval_samples_per_second": 10.094,
|
||||
"eval_sft_loss": 0.01844729855656624,
|
||||
"eval_steps_per_second": 1.289,
|
||||
"step": 35
|
||||
},
|
||||
{
|
||||
"before_init_mem_cpu": 3802071040,
|
||||
"before_init_mem_gpu": 22016,
|
||||
"epoch": 0.9807355516637478,
|
||||
"init_mem_cpu_alloc_delta": 364544,
|
||||
"init_mem_cpu_peaked_delta": 0,
|
||||
"init_mem_gpu_alloc_delta": 0,
|
||||
"init_mem_gpu_peaked_delta": 0,
|
||||
"step": 35,
|
||||
"total_flos": 39867492466688.0,
|
||||
"train_loss": 3.085822834287371,
|
||||
"train_mem_cpu_alloc_delta": 5213659136,
|
||||
"train_mem_cpu_peaked_delta": 22737326080,
|
||||
"train_mem_gpu_alloc_delta": 16267848704,
|
||||
"train_mem_gpu_peaked_delta": 36029468160,
|
||||
"train_runtime": 1628.7465,
|
||||
"train_samples_per_second": 2.805,
|
||||
"train_steps_per_second": 0.021
|
||||
}
|
||||
],
|
||||
"logging_steps": 1.0,
|
||||
"max_steps": 35,
|
||||
"num_input_tokens_seen": 0,
|
||||
"num_train_epochs": 1,
|
||||
"save_steps": 18,
|
||||
"stateful_callbacks": {
|
||||
"TrainerControl": {
|
||||
"args": {
|
||||
"should_epoch_stop": false,
|
||||
"should_evaluate": false,
|
||||
"should_log": false,
|
||||
"should_save": true,
|
||||
"should_training_stop": true
|
||||
},
|
||||
"attributes": {}
|
||||
}
|
||||
},
|
||||
"total_flos": 39867492466688.0,
|
||||
"train_batch_size": 1,
|
||||
"trial_name": null,
|
||||
"trial_params": null
|
||||
}
|
||||
3
training_args.bin
Normal file
3
training_args.bin
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:ec234bc8c9e497da6515694d4d60c171e3350788be2a96dce6d0485a99941956
|
||||
size 7800
|
||||
Reference in New Issue
Block a user