初始化项目，由ModelHub XC社区提供模型
Model: open-unlearning/tofu_Llama-2-7b-chat-hf_retain99 Source: Original Platform
This commit is contained in:
ModelHub XC
35
.gitattributes
vendored
Normal file
35
.gitattributes
vendored
Normal file
@@ -0,0 +1,35 @@
|
||||
*.7z filter=lfs diff=lfs merge=lfs -text
|
||||
*.arrow filter=lfs diff=lfs merge=lfs -text
|
||||
*.bin filter=lfs diff=lfs merge=lfs -text
|
||||
*.bz2 filter=lfs diff=lfs merge=lfs -text
|
||||
*.ckpt filter=lfs diff=lfs merge=lfs -text
|
||||
*.ftz filter=lfs diff=lfs merge=lfs -text
|
||||
*.gz filter=lfs diff=lfs merge=lfs -text
|
||||
*.h5 filter=lfs diff=lfs merge=lfs -text
|
||||
*.joblib filter=lfs diff=lfs merge=lfs -text
|
||||
*.lfs.* filter=lfs diff=lfs merge=lfs -text
|
||||
*.mlmodel filter=lfs diff=lfs merge=lfs -text
|
||||
*.model filter=lfs diff=lfs merge=lfs -text
|
||||
*.msgpack filter=lfs diff=lfs merge=lfs -text
|
||||
*.npy filter=lfs diff=lfs merge=lfs -text
|
||||
*.npz filter=lfs diff=lfs merge=lfs -text
|
||||
*.onnx filter=lfs diff=lfs merge=lfs -text
|
||||
*.ot filter=lfs diff=lfs merge=lfs -text
|
||||
*.parquet filter=lfs diff=lfs merge=lfs -text
|
||||
*.pb filter=lfs diff=lfs merge=lfs -text
|
||||
*.pickle filter=lfs diff=lfs merge=lfs -text
|
||||
*.pkl filter=lfs diff=lfs merge=lfs -text
|
||||
*.pt filter=lfs diff=lfs merge=lfs -text
|
||||
*.pth filter=lfs diff=lfs merge=lfs -text
|
||||
*.rar filter=lfs diff=lfs merge=lfs -text
|
||||
*.safetensors filter=lfs diff=lfs merge=lfs -text
|
||||
saved_model/**/* filter=lfs diff=lfs merge=lfs -text
|
||||
*.tar.* filter=lfs diff=lfs merge=lfs -text
|
||||
*.tar filter=lfs diff=lfs merge=lfs -text
|
||||
*.tflite filter=lfs diff=lfs merge=lfs -text
|
||||
*.tgz filter=lfs diff=lfs merge=lfs -text
|
||||
*.wasm filter=lfs diff=lfs merge=lfs -text
|
||||
*.xz filter=lfs diff=lfs merge=lfs -text
|
||||
*.zip filter=lfs diff=lfs merge=lfs -text
|
||||
*.zst filter=lfs diff=lfs merge=lfs -text
|
||||
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
||||
30
config.json
Normal file
30
config.json
Normal file
@@ -0,0 +1,30 @@
|
||||
{
|
||||
"_name_or_path": "meta-llama/Llama-2-7b-chat-hf",
|
||||
"architectures": [
|
||||
"LlamaForCausalLM"
|
||||
],
|
||||
"attention_bias": false,
|
||||
"attention_dropout": 0.0,
|
||||
"bos_token_id": 1,
|
||||
"eos_token_id": 2,
|
||||
"head_dim": 128,
|
||||
"hidden_act": "silu",
|
||||
"hidden_size": 4096,
|
||||
"initializer_range": 0.02,
|
||||
"intermediate_size": 11008,
|
||||
"max_position_embeddings": 4096,
|
||||
"mlp_bias": false,
|
||||
"model_type": "llama",
|
||||
"num_attention_heads": 32,
|
||||
"num_hidden_layers": 32,
|
||||
"num_key_value_heads": 32,
|
||||
"pretraining_tp": 1,
|
||||
"rms_norm_eps": 1e-05,
|
||||
"rope_scaling": null,
|
||||
"rope_theta": 10000.0,
|
||||
"tie_word_embeddings": false,
|
||||
"torch_dtype": "bfloat16",
|
||||
"transformers_version": "4.45.1",
|
||||
"use_cache": true,
|
||||
"vocab_size": 32000
|
||||
}
|
||||
10
generation_config.json
Normal file
10
generation_config.json
Normal file
@@ -0,0 +1,10 @@
|
||||
{
|
||||
"bos_token_id": 1,
|
||||
"do_sample": true,
|
||||
"eos_token_id": 2,
|
||||
"max_length": 4096,
|
||||
"pad_token_id": 0,
|
||||
"temperature": 0.6,
|
||||
"top_p": 0.9,
|
||||
"transformers_version": "4.45.1"
|
||||
}
|
||||
3
model-00001-of-00003.safetensors
Normal file
3
model-00001-of-00003.safetensors
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:8ba1f132a2e238b9d80de2fd05b79fb6de4482fcfb7739a34548ba2a14ef1c0a
|
||||
size 4938985352
|
||||
3
model-00002-of-00003.safetensors
Normal file
3
model-00002-of-00003.safetensors
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:522e270550df7b0492cb3b9a563027b554340d0088c8352ed6aedbabd3b51244
|
||||
size 4947390880
|
||||
3
model-00003-of-00003.safetensors
Normal file
3
model-00003-of-00003.safetensors
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:9ee3a6ca02b63779cd0a1a165bf7aea5748d6b92dbe601d925dfda017da6acfd
|
||||
size 3590488816
|
||||
298
model.safetensors.index.json
Normal file
298
model.safetensors.index.json
Normal file
@@ -0,0 +1,298 @@
|
||||
{
|
||||
"metadata": {
|
||||
"total_size": 13476831232
|
||||
},
|
||||
"weight_map": {
|
||||
"lm_head.weight": "model-00003-of-00003.safetensors",
|
||||
"model.embed_tokens.weight": "model-00001-of-00003.safetensors",
|
||||
"model.layers.0.input_layernorm.weight": "model-00001-of-00003.safetensors",
|
||||
"model.layers.0.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
|
||||
"model.layers.0.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
|
||||
"model.layers.0.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
|
||||
"model.layers.0.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
|
||||
"model.layers.0.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
|
||||
"model.layers.0.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
|
||||
"model.layers.0.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
|
||||
"model.layers.0.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
|
||||
"model.layers.1.input_layernorm.weight": "model-00001-of-00003.safetensors",
|
||||
"model.layers.1.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
|
||||
"model.layers.1.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
|
||||
"model.layers.1.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
|
||||
"model.layers.1.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
|
||||
"model.layers.1.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
|
||||
"model.layers.1.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
|
||||
"model.layers.1.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
|
||||
"model.layers.1.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
|
||||
"model.layers.10.input_layernorm.weight": "model-00001-of-00003.safetensors",
|
||||
"model.layers.10.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
|
||||
"model.layers.10.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
|
||||
"model.layers.10.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
|
||||
"model.layers.10.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
|
||||
"model.layers.10.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
|
||||
"model.layers.10.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
|
||||
"model.layers.10.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
|
||||
"model.layers.10.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
|
||||
"model.layers.11.input_layernorm.weight": "model-00002-of-00003.safetensors",
|
||||
"model.layers.11.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
|
||||
"model.layers.11.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
|
||||
"model.layers.11.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
|
||||
"model.layers.11.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
|
||||
"model.layers.11.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
|
||||
"model.layers.11.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
|
||||
"model.layers.11.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
|
||||
"model.layers.11.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
|
||||
"model.layers.12.input_layernorm.weight": "model-00002-of-00003.safetensors",
|
||||
"model.layers.12.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
|
||||
"model.layers.12.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
|
||||
"model.layers.12.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
|
||||
"model.layers.12.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
|
||||
"model.layers.12.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
|
||||
"model.layers.12.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
|
||||
"model.layers.12.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
|
||||
"model.layers.12.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
|
||||
"model.layers.13.input_layernorm.weight": "model-00002-of-00003.safetensors",
|
||||
"model.layers.13.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
|
||||
"model.layers.13.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
|
||||
"model.layers.13.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
|
||||
"model.layers.13.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
|
||||
"model.layers.13.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
|
||||
"model.layers.13.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
|
||||
"model.layers.13.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
|
||||
"model.layers.13.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
|
||||
"model.layers.14.input_layernorm.weight": "model-00002-of-00003.safetensors",
|
||||
"model.layers.14.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
|
||||
"model.layers.14.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
|
||||
"model.layers.14.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
|
||||
"model.layers.14.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
|
||||
"model.layers.14.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
|
||||
"model.layers.14.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
|
||||
"model.layers.14.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
|
||||
"model.layers.14.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
|
||||
"model.layers.15.input_layernorm.weight": "model-00002-of-00003.safetensors",
|
||||
"model.layers.15.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
|
||||
"model.layers.15.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
|
||||
"model.layers.15.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
|
||||
"model.layers.15.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
|
||||
"model.layers.15.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
|
||||
"model.layers.15.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
|
||||
"model.layers.15.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
|
||||
"model.layers.15.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
|
||||
"model.layers.16.input_layernorm.weight": "model-00002-of-00003.safetensors",
|
||||
"model.layers.16.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
|
||||
"model.layers.16.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
|
||||
"model.layers.16.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
|
||||
"model.layers.16.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
|
||||
"model.layers.16.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
|
||||
"model.layers.16.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
|
||||
"model.layers.16.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
|
||||
"model.layers.16.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
|
||||
"model.layers.17.input_layernorm.weight": "model-00002-of-00003.safetensors",
|
||||
"model.layers.17.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
|
||||
"model.layers.17.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
|
||||
"model.layers.17.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
|
||||
"model.layers.17.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
|
||||
"model.layers.17.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
|
||||
"model.layers.17.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
|
||||
"model.layers.17.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
|
||||
"model.layers.17.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
|
||||
"model.layers.18.input_layernorm.weight": "model-00002-of-00003.safetensors",
|
||||
"model.layers.18.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
|
||||
"model.layers.18.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
|
||||
"model.layers.18.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
|
||||
"model.layers.18.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
|
||||
"model.layers.18.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
|
||||
"model.layers.18.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
|
||||
"model.layers.18.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
|
||||
"model.layers.18.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
|
||||
"model.layers.19.input_layernorm.weight": "model-00002-of-00003.safetensors",
|
||||
"model.layers.19.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
|
||||
"model.layers.19.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
|
||||
"model.layers.19.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
|
||||
"model.layers.19.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
|
||||
"model.layers.19.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
|
||||
"model.layers.19.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
|
||||
"model.layers.19.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
|
||||
"model.layers.19.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
|
||||
"model.layers.2.input_layernorm.weight": "model-00001-of-00003.safetensors",
|
||||
"model.layers.2.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
|
||||
"model.layers.2.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
|
||||
"model.layers.2.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
|
||||
"model.layers.2.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
|
||||
"model.layers.2.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
|
||||
"model.layers.2.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
|
||||
"model.layers.2.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
|
||||
"model.layers.2.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
|
||||
"model.layers.20.input_layernorm.weight": "model-00002-of-00003.safetensors",
|
||||
"model.layers.20.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
|
||||
"model.layers.20.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
|
||||
"model.layers.20.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
|
||||
"model.layers.20.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
|
||||
"model.layers.20.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
|
||||
"model.layers.20.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
|
||||
"model.layers.20.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
|
||||
"model.layers.20.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
|
||||
"model.layers.21.input_layernorm.weight": "model-00002-of-00003.safetensors",
|
||||
"model.layers.21.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
|
||||
"model.layers.21.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
|
||||
"model.layers.21.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
|
||||
"model.layers.21.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
|
||||
"model.layers.21.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
|
||||
"model.layers.21.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
|
||||
"model.layers.21.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
|
||||
"model.layers.21.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
|
||||
"model.layers.22.input_layernorm.weight": "model-00002-of-00003.safetensors",
|
||||
"model.layers.22.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
|
||||
"model.layers.22.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
|
||||
"model.layers.22.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
|
||||
"model.layers.22.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
|
||||
"model.layers.22.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
|
||||
"model.layers.22.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
|
||||
"model.layers.22.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
|
||||
"model.layers.22.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
|
||||
"model.layers.23.input_layernorm.weight": "model-00003-of-00003.safetensors",
|
||||
"model.layers.23.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
|
||||
"model.layers.23.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
|
||||
"model.layers.23.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
|
||||
"model.layers.23.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
|
||||
"model.layers.23.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
|
||||
"model.layers.23.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
|
||||
"model.layers.23.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
|
||||
"model.layers.23.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
|
||||
"model.layers.24.input_layernorm.weight": "model-00003-of-00003.safetensors",
|
||||
"model.layers.24.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
|
||||
"model.layers.24.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
|
||||
"model.layers.24.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
|
||||
"model.layers.24.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
|
||||
"model.layers.24.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
|
||||
"model.layers.24.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
|
||||
"model.layers.24.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
|
||||
"model.layers.24.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
|
||||
"model.layers.25.input_layernorm.weight": "model-00003-of-00003.safetensors",
|
||||
"model.layers.25.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
|
||||
"model.layers.25.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
|
||||
"model.layers.25.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
|
||||
"model.layers.25.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
|
||||
"model.layers.25.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
|
||||
"model.layers.25.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
|
||||
"model.layers.25.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
|
||||
"model.layers.25.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
|
||||
"model.layers.26.input_layernorm.weight": "model-00003-of-00003.safetensors",
|
||||
"model.layers.26.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
|
||||
"model.layers.26.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
|
||||
"model.layers.26.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
|
||||
"model.layers.26.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
|
||||
"model.layers.26.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
|
||||
"model.layers.26.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
|
||||
"model.layers.26.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
|
||||
"model.layers.26.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
|
||||
"model.layers.27.input_layernorm.weight": "model-00003-of-00003.safetensors",
|
||||
"model.layers.27.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
|
||||
"model.layers.27.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
|
||||
"model.layers.27.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
|
||||
"model.layers.27.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
|
||||
"model.layers.27.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
|
||||
"model.layers.27.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
|
||||
"model.layers.27.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
|
||||
"model.layers.27.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
|
||||
"model.layers.28.input_layernorm.weight": "model-00003-of-00003.safetensors",
|
||||
"model.layers.28.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
|
||||
"model.layers.28.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
|
||||
"model.layers.28.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
|
||||
"model.layers.28.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
|
||||
"model.layers.28.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
|
||||
"model.layers.28.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
|
||||
"model.layers.28.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
|
||||
"model.layers.28.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
|
||||
"model.layers.29.input_layernorm.weight": "model-00003-of-00003.safetensors",
|
||||
"model.layers.29.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
|
||||
"model.layers.29.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
|
||||
"model.layers.29.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
|
||||
"model.layers.29.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
|
||||
"model.layers.29.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
|
||||
"model.layers.29.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
|
||||
"model.layers.29.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
|
||||
"model.layers.29.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
|
||||
"model.layers.3.input_layernorm.weight": "model-00001-of-00003.safetensors",
|
||||
"model.layers.3.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
|
||||
"model.layers.3.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
|
||||
"model.layers.3.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
|
||||
"model.layers.3.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
|
||||
"model.layers.3.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
|
||||
"model.layers.3.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
|
||||
"model.layers.3.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
|
||||
"model.layers.3.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
|
||||
"model.layers.30.input_layernorm.weight": "model-00003-of-00003.safetensors",
|
||||
"model.layers.30.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
|
||||
"model.layers.30.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
|
||||
"model.layers.30.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
|
||||
"model.layers.30.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
|
||||
"model.layers.30.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
|
||||
"model.layers.30.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
|
||||
"model.layers.30.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
|
||||
"model.layers.30.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
|
||||
"model.layers.31.input_layernorm.weight": "model-00003-of-00003.safetensors",
|
||||
"model.layers.31.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
|
||||
"model.layers.31.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
|
||||
"model.layers.31.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
|
||||
"model.layers.31.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
|
||||
"model.layers.31.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
|
||||
"model.layers.31.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
|
||||
"model.layers.31.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
|
||||
"model.layers.31.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
|
||||
"model.layers.4.input_layernorm.weight": "model-00001-of-00003.safetensors",
|
||||
"model.layers.4.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
|
||||
"model.layers.4.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
|
||||
"model.layers.4.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
|
||||
"model.layers.4.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
|
||||
"model.layers.4.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
|
||||
"model.layers.4.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
|
||||
"model.layers.4.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
|
||||
"model.layers.4.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
|
||||
"model.layers.5.input_layernorm.weight": "model-00001-of-00003.safetensors",
|
||||
"model.layers.5.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
|
||||
"model.layers.5.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
|
||||
"model.layers.5.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
|
||||
"model.layers.5.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
|
||||
"model.layers.5.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
|
||||
"model.layers.5.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
|
||||
"model.layers.5.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
|
||||
"model.layers.5.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
|
||||
"model.layers.6.input_layernorm.weight": "model-00001-of-00003.safetensors",
|
||||
"model.layers.6.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
|
||||
"model.layers.6.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
|
||||
"model.layers.6.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
|
||||
"model.layers.6.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
|
||||
"model.layers.6.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
|
||||
"model.layers.6.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
|
||||
"model.layers.6.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
|
||||
"model.layers.6.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
|
||||
"model.layers.7.input_layernorm.weight": "model-00001-of-00003.safetensors",
|
||||
"model.layers.7.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
|
||||
"model.layers.7.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
|
||||
"model.layers.7.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
|
||||
"model.layers.7.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
|
||||
"model.layers.7.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
|
||||
"model.layers.7.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
|
||||
"model.layers.7.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
|
||||
"model.layers.7.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
|
||||
"model.layers.8.input_layernorm.weight": "model-00001-of-00003.safetensors",
|
||||
"model.layers.8.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
|
||||
"model.layers.8.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
|
||||
"model.layers.8.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
|
||||
"model.layers.8.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
|
||||
"model.layers.8.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
|
||||
"model.layers.8.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
|
||||
"model.layers.8.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
|
||||
"model.layers.8.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
|
||||
"model.layers.9.input_layernorm.weight": "model-00001-of-00003.safetensors",
|
||||
"model.layers.9.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
|
||||
"model.layers.9.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
|
||||
"model.layers.9.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
|
||||
"model.layers.9.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
|
||||
"model.layers.9.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
|
||||
"model.layers.9.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
|
||||
"model.layers.9.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
|
||||
"model.layers.9.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
|
||||
"model.norm.weight": "model-00003-of-00003.safetensors"
|
||||
}
|
||||
}
|
||||
24
special_tokens_map.json
Normal file
24
special_tokens_map.json
Normal file
@@ -0,0 +1,24 @@
|
||||
{
|
||||
"bos_token": {
|
||||
"content": "<s>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false
|
||||
},
|
||||
"eos_token": {
|
||||
"content": "</s>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false
|
||||
},
|
||||
"pad_token": "</s>",
|
||||
"unk_token": {
|
||||
"content": "<unk>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false
|
||||
}
|
||||
}
|
||||
277144
tokenizer.json
Normal file
277144
tokenizer.json
Normal file
File diff suppressed because it is too large
Load Diff
43
tokenizer_config.json
Normal file
43
tokenizer_config.json
Normal file
@@ -0,0 +1,43 @@
|
||||
{
|
||||
"add_bos_token": true,
|
||||
"add_eos_token": false,
|
||||
"add_prefix_space": null,
|
||||
"added_tokens_decoder": {
|
||||
"0": {
|
||||
"content": "<unk>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"1": {
|
||||
"content": "<s>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"2": {
|
||||
"content": "</s>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
}
|
||||
},
|
||||
"bos_token": "<s>",
|
||||
"chat_template": "{% if messages[0]['role'] == 'system' %}{% set loop_messages = messages[1:] %}{% set system_message = messages[0]['content'] %}{% else %}{% set loop_messages = messages %}{% set system_message = false %}{% endif %}{% for message in loop_messages %}{% if (message['role'] == 'user') != (loop.index0 % 2 == 0) %}{{ raise_exception('Conversation roles must alternate user/assistant/user/assistant/...') }}{% endif %}{% if loop.index0 == 0 and system_message != false %}{% set content = '<<SYS>>\\n' + system_message + '\\n<</SYS>>\\n\\n' + message['content'] %}{% else %}{% set content = message['content'] %}{% endif %}{% if message['role'] == 'user' %}{{ bos_token + '[INST] ' + content.strip() + ' [/INST]' }}{% elif message['role'] == 'assistant' %}{{ ' ' + content.strip() + ' ' + eos_token }}{% endif %}{% endfor %}",
|
||||
"clean_up_tokenization_spaces": false,
|
||||
"eos_token": "</s>",
|
||||
"legacy": false,
|
||||
"model_max_length": 1000000000000000019884624838656,
|
||||
"pad_token": "</s>",
|
||||
"padding_side": "right",
|
||||
"sp_model_kwargs": {},
|
||||
"tokenizer_class": "LlamaTokenizer",
|
||||
"unk_token": "<unk>",
|
||||
"use_default_system_prompt": false
|
||||
}
|
||||
903
trainer_state.json
Normal file
903
trainer_state.json
Normal file
@@ -0,0 +1,903 @@
|
||||
{
|
||||
"best_metric": null,
|
||||
"best_model_checkpoint": null,
|
||||
"epoch": 4.96969696969697,
|
||||
"eval_steps": 500,
|
||||
"global_step": 615,
|
||||
"is_hyper_param_search": false,
|
||||
"is_local_process_zero": true,
|
||||
"is_world_process_zero": true,
|
||||
"log_history": [
|
||||
{
|
||||
"epoch": 0.04040404040404041,
|
||||
"grad_norm": 31.30476482142413,
|
||||
"learning_rate": 4.0650406504065046e-07,
|
||||
"loss": 2.731,
|
||||
"step": 5
|
||||
},
|
||||
{
|
||||
"epoch": 0.08080808080808081,
|
||||
"grad_norm": 34.876304019127446,
|
||||
"learning_rate": 8.130081300813009e-07,
|
||||
"loss": 2.7314,
|
||||
"step": 10
|
||||
},
|
||||
{
|
||||
"epoch": 0.12121212121212122,
|
||||
"grad_norm": 30.38046670226559,
|
||||
"learning_rate": 1.2195121951219514e-06,
|
||||
"loss": 2.563,
|
||||
"step": 15
|
||||
},
|
||||
{
|
||||
"epoch": 0.16161616161616163,
|
||||
"grad_norm": 22.736370534804927,
|
||||
"learning_rate": 1.6260162601626018e-06,
|
||||
"loss": 2.1967,
|
||||
"step": 20
|
||||
},
|
||||
{
|
||||
"epoch": 0.20202020202020202,
|
||||
"grad_norm": 8.559531969065942,
|
||||
"learning_rate": 2.0325203252032523e-06,
|
||||
"loss": 1.8438,
|
||||
"step": 25
|
||||
},
|
||||
{
|
||||
"epoch": 0.24242424242424243,
|
||||
"grad_norm": 9.33384999992704,
|
||||
"learning_rate": 2.4390243902439027e-06,
|
||||
"loss": 1.747,
|
||||
"step": 30
|
||||
},
|
||||
{
|
||||
"epoch": 0.2828282828282828,
|
||||
"grad_norm": 7.752314828812326,
|
||||
"learning_rate": 2.845528455284553e-06,
|
||||
"loss": 1.6554,
|
||||
"step": 35
|
||||
},
|
||||
{
|
||||
"epoch": 0.32323232323232326,
|
||||
"grad_norm": 8.28616324449917,
|
||||
"learning_rate": 3.2520325203252037e-06,
|
||||
"loss": 1.5524,
|
||||
"step": 40
|
||||
},
|
||||
{
|
||||
"epoch": 0.36363636363636365,
|
||||
"grad_norm": 7.1680338706831135,
|
||||
"learning_rate": 3.6585365853658537e-06,
|
||||
"loss": 1.504,
|
||||
"step": 45
|
||||
},
|
||||
{
|
||||
"epoch": 0.40404040404040403,
|
||||
"grad_norm": 4.72541363138399,
|
||||
"learning_rate": 4.0650406504065046e-06,
|
||||
"loss": 1.3992,
|
||||
"step": 50
|
||||
},
|
||||
{
|
||||
"epoch": 0.4444444444444444,
|
||||
"grad_norm": 4.997709508558392,
|
||||
"learning_rate": 4.471544715447155e-06,
|
||||
"loss": 1.4008,
|
||||
"step": 55
|
||||
},
|
||||
{
|
||||
"epoch": 0.48484848484848486,
|
||||
"grad_norm": 4.732833533844343,
|
||||
"learning_rate": 4.8780487804878055e-06,
|
||||
"loss": 1.3928,
|
||||
"step": 60
|
||||
},
|
||||
{
|
||||
"epoch": 0.5252525252525253,
|
||||
"grad_norm": 4.632800812924082,
|
||||
"learning_rate": 5.2845528455284555e-06,
|
||||
"loss": 1.3891,
|
||||
"step": 65
|
||||
},
|
||||
{
|
||||
"epoch": 0.5656565656565656,
|
||||
"grad_norm": 4.4532404789711135,
|
||||
"learning_rate": 5.691056910569106e-06,
|
||||
"loss": 1.35,
|
||||
"step": 70
|
||||
},
|
||||
{
|
||||
"epoch": 0.6060606060606061,
|
||||
"grad_norm": 4.210508341423186,
|
||||
"learning_rate": 6.0975609756097564e-06,
|
||||
"loss": 1.3411,
|
||||
"step": 75
|
||||
},
|
||||
{
|
||||
"epoch": 0.6464646464646465,
|
||||
"grad_norm": 4.225418310599549,
|
||||
"learning_rate": 6.504065040650407e-06,
|
||||
"loss": 1.3076,
|
||||
"step": 80
|
||||
},
|
||||
{
|
||||
"epoch": 0.6868686868686869,
|
||||
"grad_norm": 4.283561944553204,
|
||||
"learning_rate": 6.910569105691057e-06,
|
||||
"loss": 1.329,
|
||||
"step": 85
|
||||
},
|
||||
{
|
||||
"epoch": 0.7272727272727273,
|
||||
"grad_norm": 4.660669898211197,
|
||||
"learning_rate": 7.317073170731707e-06,
|
||||
"loss": 1.2887,
|
||||
"step": 90
|
||||
},
|
||||
{
|
||||
"epoch": 0.7676767676767676,
|
||||
"grad_norm": 6.089953972521491,
|
||||
"learning_rate": 7.723577235772358e-06,
|
||||
"loss": 1.3019,
|
||||
"step": 95
|
||||
},
|
||||
{
|
||||
"epoch": 0.8080808080808081,
|
||||
"grad_norm": 4.546928403826633,
|
||||
"learning_rate": 8.130081300813009e-06,
|
||||
"loss": 1.2888,
|
||||
"step": 100
|
||||
},
|
||||
{
|
||||
"epoch": 0.8484848484848485,
|
||||
"grad_norm": 4.685274208183413,
|
||||
"learning_rate": 8.536585365853658e-06,
|
||||
"loss": 1.281,
|
||||
"step": 105
|
||||
},
|
||||
{
|
||||
"epoch": 0.8888888888888888,
|
||||
"grad_norm": 5.72827065904213,
|
||||
"learning_rate": 8.94308943089431e-06,
|
||||
"loss": 1.232,
|
||||
"step": 110
|
||||
},
|
||||
{
|
||||
"epoch": 0.9292929292929293,
|
||||
"grad_norm": 4.977185099228744,
|
||||
"learning_rate": 9.34959349593496e-06,
|
||||
"loss": 1.2568,
|
||||
"step": 115
|
||||
},
|
||||
{
|
||||
"epoch": 0.9696969696969697,
|
||||
"grad_norm": 4.9133334320337525,
|
||||
"learning_rate": 9.756097560975611e-06,
|
||||
"loss": 1.2952,
|
||||
"step": 120
|
||||
},
|
||||
{
|
||||
"epoch": 1.0101010101010102,
|
||||
"grad_norm": 3.947650506403009,
|
||||
"learning_rate": 9.959349593495936e-06,
|
||||
"loss": 1.2198,
|
||||
"step": 125
|
||||
},
|
||||
{
|
||||
"epoch": 1.0505050505050506,
|
||||
"grad_norm": 5.028654870968992,
|
||||
"learning_rate": 9.857723577235772e-06,
|
||||
"loss": 0.9984,
|
||||
"step": 130
|
||||
},
|
||||
{
|
||||
"epoch": 1.0909090909090908,
|
||||
"grad_norm": 4.471104202424854,
|
||||
"learning_rate": 9.756097560975611e-06,
|
||||
"loss": 0.9632,
|
||||
"step": 135
|
||||
},
|
||||
{
|
||||
"epoch": 1.1313131313131313,
|
||||
"grad_norm": 4.493632071416398,
|
||||
"learning_rate": 9.654471544715448e-06,
|
||||
"loss": 1.011,
|
||||
"step": 140
|
||||
},
|
||||
{
|
||||
"epoch": 1.1717171717171717,
|
||||
"grad_norm": 5.3285602333513795,
|
||||
"learning_rate": 9.552845528455286e-06,
|
||||
"loss": 1.0111,
|
||||
"step": 145
|
||||
},
|
||||
{
|
||||
"epoch": 1.2121212121212122,
|
||||
"grad_norm": 4.70651395172554,
|
||||
"learning_rate": 9.451219512195122e-06,
|
||||
"loss": 1.0664,
|
||||
"step": 150
|
||||
},
|
||||
{
|
||||
"epoch": 1.2525252525252526,
|
||||
"grad_norm": 5.029721194822212,
|
||||
"learning_rate": 9.34959349593496e-06,
|
||||
"loss": 1.0179,
|
||||
"step": 155
|
||||
},
|
||||
{
|
||||
"epoch": 1.2929292929292928,
|
||||
"grad_norm": 4.913503297434893,
|
||||
"learning_rate": 9.247967479674797e-06,
|
||||
"loss": 1.0082,
|
||||
"step": 160
|
||||
},
|
||||
{
|
||||
"epoch": 1.3333333333333333,
|
||||
"grad_norm": 4.896160083472389,
|
||||
"learning_rate": 9.146341463414635e-06,
|
||||
"loss": 0.9799,
|
||||
"step": 165
|
||||
},
|
||||
{
|
||||
"epoch": 1.3737373737373737,
|
||||
"grad_norm": 5.294928046006384,
|
||||
"learning_rate": 9.044715447154472e-06,
|
||||
"loss": 0.9937,
|
||||
"step": 170
|
||||
},
|
||||
{
|
||||
"epoch": 1.4141414141414141,
|
||||
"grad_norm": 5.075223633830589,
|
||||
"learning_rate": 8.94308943089431e-06,
|
||||
"loss": 0.9865,
|
||||
"step": 175
|
||||
},
|
||||
{
|
||||
"epoch": 1.4545454545454546,
|
||||
"grad_norm": 5.618687705537486,
|
||||
"learning_rate": 8.841463414634148e-06,
|
||||
"loss": 0.9766,
|
||||
"step": 180
|
||||
},
|
||||
{
|
||||
"epoch": 1.494949494949495,
|
||||
"grad_norm": 4.623192938074825,
|
||||
"learning_rate": 8.739837398373985e-06,
|
||||
"loss": 0.9424,
|
||||
"step": 185
|
||||
},
|
||||
{
|
||||
"epoch": 1.5353535353535355,
|
||||
"grad_norm": 5.386337630316101,
|
||||
"learning_rate": 8.638211382113821e-06,
|
||||
"loss": 0.9773,
|
||||
"step": 190
|
||||
},
|
||||
{
|
||||
"epoch": 1.5757575757575757,
|
||||
"grad_norm": 5.672065405501361,
|
||||
"learning_rate": 8.536585365853658e-06,
|
||||
"loss": 0.9462,
|
||||
"step": 195
|
||||
},
|
||||
{
|
||||
"epoch": 1.6161616161616161,
|
||||
"grad_norm": 4.761077656696054,
|
||||
"learning_rate": 8.434959349593497e-06,
|
||||
"loss": 0.9518,
|
||||
"step": 200
|
||||
},
|
||||
{
|
||||
"epoch": 1.6565656565656566,
|
||||
"grad_norm": 4.531676072464644,
|
||||
"learning_rate": 8.333333333333334e-06,
|
||||
"loss": 0.9213,
|
||||
"step": 205
|
||||
},
|
||||
{
|
||||
"epoch": 1.696969696969697,
|
||||
"grad_norm": 6.266355273048769,
|
||||
"learning_rate": 8.23170731707317e-06,
|
||||
"loss": 0.9721,
|
||||
"step": 210
|
||||
},
|
||||
{
|
||||
"epoch": 1.7373737373737375,
|
||||
"grad_norm": 5.236443633591077,
|
||||
"learning_rate": 8.130081300813009e-06,
|
||||
"loss": 0.9725,
|
||||
"step": 215
|
||||
},
|
||||
{
|
||||
"epoch": 1.7777777777777777,
|
||||
"grad_norm": 4.648565215297412,
|
||||
"learning_rate": 8.028455284552846e-06,
|
||||
"loss": 0.9775,
|
||||
"step": 220
|
||||
},
|
||||
{
|
||||
"epoch": 1.8181818181818183,
|
||||
"grad_norm": 4.6401670229714105,
|
||||
"learning_rate": 7.926829268292685e-06,
|
||||
"loss": 0.9329,
|
||||
"step": 225
|
||||
},
|
||||
{
|
||||
"epoch": 1.8585858585858586,
|
||||
"grad_norm": 5.19850533785999,
|
||||
"learning_rate": 7.82520325203252e-06,
|
||||
"loss": 0.9024,
|
||||
"step": 230
|
||||
},
|
||||
{
|
||||
"epoch": 1.898989898989899,
|
||||
"grad_norm": 4.582681987215693,
|
||||
"learning_rate": 7.723577235772358e-06,
|
||||
"loss": 0.9759,
|
||||
"step": 235
|
||||
},
|
||||
{
|
||||
"epoch": 1.9393939393939394,
|
||||
"grad_norm": 4.73345445713379,
|
||||
"learning_rate": 7.621951219512196e-06,
|
||||
"loss": 0.9209,
|
||||
"step": 240
|
||||
},
|
||||
{
|
||||
"epoch": 1.9797979797979797,
|
||||
"grad_norm": 5.139747254586091,
|
||||
"learning_rate": 7.520325203252034e-06,
|
||||
"loss": 0.957,
|
||||
"step": 245
|
||||
},
|
||||
{
|
||||
"epoch": 2.0202020202020203,
|
||||
"grad_norm": 4.422348917224151,
|
||||
"learning_rate": 7.41869918699187e-06,
|
||||
"loss": 0.7453,
|
||||
"step": 250
|
||||
},
|
||||
{
|
||||
"epoch": 2.0606060606060606,
|
||||
"grad_norm": 4.433867056483005,
|
||||
"learning_rate": 7.317073170731707e-06,
|
||||
"loss": 0.4785,
|
||||
"step": 255
|
||||
},
|
||||
{
|
||||
"epoch": 2.101010101010101,
|
||||
"grad_norm": 5.90566178911042,
|
||||
"learning_rate": 7.215447154471545e-06,
|
||||
"loss": 0.4377,
|
||||
"step": 260
|
||||
},
|
||||
{
|
||||
"epoch": 2.1414141414141414,
|
||||
"grad_norm": 5.260653029839055,
|
||||
"learning_rate": 7.113821138211383e-06,
|
||||
"loss": 0.4643,
|
||||
"step": 265
|
||||
},
|
||||
{
|
||||
"epoch": 2.1818181818181817,
|
||||
"grad_norm": 4.612238596225972,
|
||||
"learning_rate": 7.01219512195122e-06,
|
||||
"loss": 0.404,
|
||||
"step": 270
|
||||
},
|
||||
{
|
||||
"epoch": 2.2222222222222223,
|
||||
"grad_norm": 4.45300883866109,
|
||||
"learning_rate": 6.910569105691057e-06,
|
||||
"loss": 0.4569,
|
||||
"step": 275
|
||||
},
|
||||
{
|
||||
"epoch": 2.2626262626262625,
|
||||
"grad_norm": 6.5459475201379185,
|
||||
"learning_rate": 6.808943089430895e-06,
|
||||
"loss": 0.4713,
|
||||
"step": 280
|
||||
},
|
||||
{
|
||||
"epoch": 2.303030303030303,
|
||||
"grad_norm": 6.029714852053309,
|
||||
"learning_rate": 6.707317073170733e-06,
|
||||
"loss": 0.4235,
|
||||
"step": 285
|
||||
},
|
||||
{
|
||||
"epoch": 2.3434343434343434,
|
||||
"grad_norm": 5.787746448522448,
|
||||
"learning_rate": 6.60569105691057e-06,
|
||||
"loss": 0.4743,
|
||||
"step": 290
|
||||
},
|
||||
{
|
||||
"epoch": 2.3838383838383836,
|
||||
"grad_norm": 5.3235709402562295,
|
||||
"learning_rate": 6.504065040650407e-06,
|
||||
"loss": 0.4456,
|
||||
"step": 295
|
||||
},
|
||||
{
|
||||
"epoch": 2.4242424242424243,
|
||||
"grad_norm": 5.307124582254565,
|
||||
"learning_rate": 6.402439024390244e-06,
|
||||
"loss": 0.4547,
|
||||
"step": 300
|
||||
},
|
||||
{
|
||||
"epoch": 2.4646464646464645,
|
||||
"grad_norm": 4.7529611960917455,
|
||||
"learning_rate": 6.300813008130082e-06,
|
||||
"loss": 0.4561,
|
||||
"step": 305
|
||||
},
|
||||
{
|
||||
"epoch": 2.505050505050505,
|
||||
"grad_norm": 5.798729033666169,
|
||||
"learning_rate": 6.199186991869919e-06,
|
||||
"loss": 0.4436,
|
||||
"step": 310
|
||||
},
|
||||
{
|
||||
"epoch": 2.5454545454545454,
|
||||
"grad_norm": 4.7847243271957804,
|
||||
"learning_rate": 6.0975609756097564e-06,
|
||||
"loss": 0.451,
|
||||
"step": 315
|
||||
},
|
||||
{
|
||||
"epoch": 2.5858585858585856,
|
||||
"grad_norm": 5.4890813350191685,
|
||||
"learning_rate": 5.995934959349594e-06,
|
||||
"loss": 0.4518,
|
||||
"step": 320
|
||||
},
|
||||
{
|
||||
"epoch": 2.6262626262626263,
|
||||
"grad_norm": 5.682801531435368,
|
||||
"learning_rate": 5.894308943089432e-06,
|
||||
"loss": 0.4545,
|
||||
"step": 325
|
||||
},
|
||||
{
|
||||
"epoch": 2.6666666666666665,
|
||||
"grad_norm": 5.903369254781947,
|
||||
"learning_rate": 5.792682926829269e-06,
|
||||
"loss": 0.4553,
|
||||
"step": 330
|
||||
},
|
||||
{
|
||||
"epoch": 2.707070707070707,
|
||||
"grad_norm": 5.173960592573969,
|
||||
"learning_rate": 5.691056910569106e-06,
|
||||
"loss": 0.4451,
|
||||
"step": 335
|
||||
},
|
||||
{
|
||||
"epoch": 2.7474747474747474,
|
||||
"grad_norm": 5.151186209056585,
|
||||
"learning_rate": 5.589430894308944e-06,
|
||||
"loss": 0.4425,
|
||||
"step": 340
|
||||
},
|
||||
{
|
||||
"epoch": 2.787878787878788,
|
||||
"grad_norm": 5.485953908480792,
|
||||
"learning_rate": 5.487804878048781e-06,
|
||||
"loss": 0.4704,
|
||||
"step": 345
|
||||
},
|
||||
{
|
||||
"epoch": 2.8282828282828283,
|
||||
"grad_norm": 5.020350074243234,
|
||||
"learning_rate": 5.386178861788618e-06,
|
||||
"loss": 0.4904,
|
||||
"step": 350
|
||||
},
|
||||
{
|
||||
"epoch": 2.8686868686868685,
|
||||
"grad_norm": 4.984806682884462,
|
||||
"learning_rate": 5.2845528455284555e-06,
|
||||
"loss": 0.4489,
|
||||
"step": 355
|
||||
},
|
||||
{
|
||||
"epoch": 2.909090909090909,
|
||||
"grad_norm": 5.1865191320521635,
|
||||
"learning_rate": 5.182926829268293e-06,
|
||||
"loss": 0.4657,
|
||||
"step": 360
|
||||
},
|
||||
{
|
||||
"epoch": 2.9494949494949494,
|
||||
"grad_norm": 4.984202536955517,
|
||||
"learning_rate": 5.081300813008131e-06,
|
||||
"loss": 0.4495,
|
||||
"step": 365
|
||||
},
|
||||
{
|
||||
"epoch": 2.98989898989899,
|
||||
"grad_norm": 6.161977258151849,
|
||||
"learning_rate": 4.979674796747968e-06,
|
||||
"loss": 0.4654,
|
||||
"step": 370
|
||||
},
|
||||
{
|
||||
"epoch": 3.0303030303030303,
|
||||
"grad_norm": 3.5760517296663625,
|
||||
"learning_rate": 4.8780487804878055e-06,
|
||||
"loss": 0.2076,
|
||||
"step": 375
|
||||
},
|
||||
{
|
||||
"epoch": 3.0707070707070705,
|
||||
"grad_norm": 4.247871677624529,
|
||||
"learning_rate": 4.776422764227643e-06,
|
||||
"loss": 0.1167,
|
||||
"step": 380
|
||||
},
|
||||
{
|
||||
"epoch": 3.111111111111111,
|
||||
"grad_norm": 5.39016163716647,
|
||||
"learning_rate": 4.67479674796748e-06,
|
||||
"loss": 0.1218,
|
||||
"step": 385
|
||||
},
|
||||
{
|
||||
"epoch": 3.1515151515151514,
|
||||
"grad_norm": 4.225348592606848,
|
||||
"learning_rate": 4.573170731707318e-06,
|
||||
"loss": 0.1122,
|
||||
"step": 390
|
||||
},
|
||||
{
|
||||
"epoch": 3.191919191919192,
|
||||
"grad_norm": 4.127059046967633,
|
||||
"learning_rate": 4.471544715447155e-06,
|
||||
"loss": 0.1161,
|
||||
"step": 395
|
||||
},
|
||||
{
|
||||
"epoch": 3.2323232323232323,
|
||||
"grad_norm": 5.332978564913852,
|
||||
"learning_rate": 4.369918699186992e-06,
|
||||
"loss": 0.1199,
|
||||
"step": 400
|
||||
},
|
||||
{
|
||||
"epoch": 3.2727272727272725,
|
||||
"grad_norm": 4.720104310845255,
|
||||
"learning_rate": 4.268292682926829e-06,
|
||||
"loss": 0.112,
|
||||
"step": 405
|
||||
},
|
||||
{
|
||||
"epoch": 3.313131313131313,
|
||||
"grad_norm": 4.208703942316227,
|
||||
"learning_rate": 4.166666666666667e-06,
|
||||
"loss": 0.1005,
|
||||
"step": 410
|
||||
},
|
||||
{
|
||||
"epoch": 3.3535353535353534,
|
||||
"grad_norm": 3.8232230762287966,
|
||||
"learning_rate": 4.0650406504065046e-06,
|
||||
"loss": 0.1224,
|
||||
"step": 415
|
||||
},
|
||||
{
|
||||
"epoch": 3.393939393939394,
|
||||
"grad_norm": 4.5434075590160665,
|
||||
"learning_rate": 3.963414634146342e-06,
|
||||
"loss": 0.1103,
|
||||
"step": 420
|
||||
},
|
||||
{
|
||||
"epoch": 3.4343434343434343,
|
||||
"grad_norm": 3.754529149160335,
|
||||
"learning_rate": 3.861788617886179e-06,
|
||||
"loss": 0.1103,
|
||||
"step": 425
|
||||
},
|
||||
{
|
||||
"epoch": 3.474747474747475,
|
||||
"grad_norm": 3.9672869856555413,
|
||||
"learning_rate": 3.760162601626017e-06,
|
||||
"loss": 0.1139,
|
||||
"step": 430
|
||||
},
|
||||
{
|
||||
"epoch": 3.515151515151515,
|
||||
"grad_norm": 3.991324257907671,
|
||||
"learning_rate": 3.6585365853658537e-06,
|
||||
"loss": 0.1022,
|
||||
"step": 435
|
||||
},
|
||||
{
|
||||
"epoch": 3.5555555555555554,
|
||||
"grad_norm": 5.199667387412245,
|
||||
"learning_rate": 3.5569105691056914e-06,
|
||||
"loss": 0.1086,
|
||||
"step": 440
|
||||
},
|
||||
{
|
||||
"epoch": 3.595959595959596,
|
||||
"grad_norm": 4.050131042131313,
|
||||
"learning_rate": 3.4552845528455287e-06,
|
||||
"loss": 0.1106,
|
||||
"step": 445
|
||||
},
|
||||
{
|
||||
"epoch": 3.6363636363636362,
|
||||
"grad_norm": 4.153390414438305,
|
||||
"learning_rate": 3.3536585365853664e-06,
|
||||
"loss": 0.0984,
|
||||
"step": 450
|
||||
},
|
||||
{
|
||||
"epoch": 3.676767676767677,
|
||||
"grad_norm": 4.202538793062038,
|
||||
"learning_rate": 3.2520325203252037e-06,
|
||||
"loss": 0.1084,
|
||||
"step": 455
|
||||
},
|
||||
{
|
||||
"epoch": 3.717171717171717,
|
||||
"grad_norm": 4.048115883343879,
|
||||
"learning_rate": 3.150406504065041e-06,
|
||||
"loss": 0.1145,
|
||||
"step": 460
|
||||
},
|
||||
{
|
||||
"epoch": 3.757575757575758,
|
||||
"grad_norm": 4.0191610893455625,
|
||||
"learning_rate": 3.0487804878048782e-06,
|
||||
"loss": 0.1024,
|
||||
"step": 465
|
||||
},
|
||||
{
|
||||
"epoch": 3.797979797979798,
|
||||
"grad_norm": 4.392055298743119,
|
||||
"learning_rate": 2.947154471544716e-06,
|
||||
"loss": 0.1143,
|
||||
"step": 470
|
||||
},
|
||||
{
|
||||
"epoch": 3.8383838383838382,
|
||||
"grad_norm": 4.613838678121089,
|
||||
"learning_rate": 2.845528455284553e-06,
|
||||
"loss": 0.0903,
|
||||
"step": 475
|
||||
},
|
||||
{
|
||||
"epoch": 3.878787878787879,
|
||||
"grad_norm": 4.4543588844155995,
|
||||
"learning_rate": 2.7439024390243905e-06,
|
||||
"loss": 0.1089,
|
||||
"step": 480
|
||||
},
|
||||
{
|
||||
"epoch": 3.919191919191919,
|
||||
"grad_norm": 4.624868129432924,
|
||||
"learning_rate": 2.6422764227642278e-06,
|
||||
"loss": 0.0916,
|
||||
"step": 485
|
||||
},
|
||||
{
|
||||
"epoch": 3.9595959595959593,
|
||||
"grad_norm": 6.1365151922374555,
|
||||
"learning_rate": 2.5406504065040655e-06,
|
||||
"loss": 0.1023,
|
||||
"step": 490
|
||||
},
|
||||
{
|
||||
"epoch": 4.0,
|
||||
"grad_norm": 4.628449029625847,
|
||||
"learning_rate": 2.4390243902439027e-06,
|
||||
"loss": 0.1155,
|
||||
"step": 495
|
||||
},
|
||||
{
|
||||
"epoch": 4.040404040404041,
|
||||
"grad_norm": 2.150878593783259,
|
||||
"learning_rate": 2.33739837398374e-06,
|
||||
"loss": 0.0283,
|
||||
"step": 500
|
||||
},
|
||||
{
|
||||
"epoch": 4.08080808080808,
|
||||
"grad_norm": 2.492419372701859,
|
||||
"learning_rate": 2.2357723577235773e-06,
|
||||
"loss": 0.0253,
|
||||
"step": 505
|
||||
},
|
||||
{
|
||||
"epoch": 4.121212121212121,
|
||||
"grad_norm": 1.5843746130365612,
|
||||
"learning_rate": 2.1341463414634146e-06,
|
||||
"loss": 0.0268,
|
||||
"step": 510
|
||||
},
|
||||
{
|
||||
"epoch": 4.161616161616162,
|
||||
"grad_norm": 2.9205816632118444,
|
||||
"learning_rate": 2.0325203252032523e-06,
|
||||
"loss": 0.0216,
|
||||
"step": 515
|
||||
},
|
||||
{
|
||||
"epoch": 4.202020202020202,
|
||||
"grad_norm": 2.0649905754681264,
|
||||
"learning_rate": 1.9308943089430896e-06,
|
||||
"loss": 0.0202,
|
||||
"step": 520
|
||||
},
|
||||
{
|
||||
"epoch": 4.242424242424242,
|
||||
"grad_norm": 3.069265957408005,
|
||||
"learning_rate": 1.8292682926829268e-06,
|
||||
"loss": 0.0214,
|
||||
"step": 525
|
||||
},
|
||||
{
|
||||
"epoch": 4.282828282828283,
|
||||
"grad_norm": 1.9315088953348416,
|
||||
"learning_rate": 1.7276422764227643e-06,
|
||||
"loss": 0.0224,
|
||||
"step": 530
|
||||
},
|
||||
{
|
||||
"epoch": 4.3232323232323235,
|
||||
"grad_norm": 2.077673475013364,
|
||||
"learning_rate": 1.6260162601626018e-06,
|
||||
"loss": 0.0238,
|
||||
"step": 535
|
||||
},
|
||||
{
|
||||
"epoch": 4.363636363636363,
|
||||
"grad_norm": 2.3272228657320633,
|
||||
"learning_rate": 1.5243902439024391e-06,
|
||||
"loss": 0.0227,
|
||||
"step": 540
|
||||
},
|
||||
{
|
||||
"epoch": 4.404040404040404,
|
||||
"grad_norm": 2.083971492908658,
|
||||
"learning_rate": 1.4227642276422766e-06,
|
||||
"loss": 0.0256,
|
||||
"step": 545
|
||||
},
|
||||
{
|
||||
"epoch": 4.444444444444445,
|
||||
"grad_norm": 2.0472174007391097,
|
||||
"learning_rate": 1.3211382113821139e-06,
|
||||
"loss": 0.0222,
|
||||
"step": 550
|
||||
},
|
||||
{
|
||||
"epoch": 4.484848484848484,
|
||||
"grad_norm": 2.2109189062342565,
|
||||
"learning_rate": 1.2195121951219514e-06,
|
||||
"loss": 0.023,
|
||||
"step": 555
|
||||
},
|
||||
{
|
||||
"epoch": 4.525252525252525,
|
||||
"grad_norm": 1.112819881837253,
|
||||
"learning_rate": 1.1178861788617887e-06,
|
||||
"loss": 0.0257,
|
||||
"step": 560
|
||||
},
|
||||
{
|
||||
"epoch": 4.565656565656566,
|
||||
"grad_norm": 0.8855575737373936,
|
||||
"learning_rate": 1.0162601626016261e-06,
|
||||
"loss": 0.0166,
|
||||
"step": 565
|
||||
},
|
||||
{
|
||||
"epoch": 4.606060606060606,
|
||||
"grad_norm": 1.690338395060282,
|
||||
"learning_rate": 9.146341463414634e-07,
|
||||
"loss": 0.0177,
|
||||
"step": 570
|
||||
},
|
||||
{
|
||||
"epoch": 4.646464646464646,
|
||||
"grad_norm": 2.9632288047181983,
|
||||
"learning_rate": 8.130081300813009e-07,
|
||||
"loss": 0.0193,
|
||||
"step": 575
|
||||
},
|
||||
{
|
||||
"epoch": 4.686868686868687,
|
||||
"grad_norm": 1.440170472271606,
|
||||
"learning_rate": 7.113821138211383e-07,
|
||||
"loss": 0.0228,
|
||||
"step": 580
|
||||
},
|
||||
{
|
||||
"epoch": 4.7272727272727275,
|
||||
"grad_norm": 1.2155257751361124,
|
||||
"learning_rate": 6.097560975609757e-07,
|
||||
"loss": 0.0182,
|
||||
"step": 585
|
||||
},
|
||||
{
|
||||
"epoch": 4.767676767676767,
|
||||
"grad_norm": 1.6706742205751832,
|
||||
"learning_rate": 5.081300813008131e-07,
|
||||
"loss": 0.0219,
|
||||
"step": 590
|
||||
},
|
||||
{
|
||||
"epoch": 4.808080808080808,
|
||||
"grad_norm": 1.879246307640821,
|
||||
"learning_rate": 4.0650406504065046e-07,
|
||||
"loss": 0.018,
|
||||
"step": 595
|
||||
},
|
||||
{
|
||||
"epoch": 4.848484848484849,
|
||||
"grad_norm": 2.6413853266749987,
|
||||
"learning_rate": 3.0487804878048784e-07,
|
||||
"loss": 0.017,
|
||||
"step": 600
|
||||
},
|
||||
{
|
||||
"epoch": 4.888888888888889,
|
||||
"grad_norm": 1.9369171496463586,
|
||||
"learning_rate": 2.0325203252032523e-07,
|
||||
"loss": 0.018,
|
||||
"step": 605
|
||||
},
|
||||
{
|
||||
"epoch": 4.929292929292929,
|
||||
"grad_norm": 1.467255552769616,
|
||||
"learning_rate": 1.0162601626016261e-07,
|
||||
"loss": 0.0181,
|
||||
"step": 610
|
||||
},
|
||||
{
|
||||
"epoch": 4.96969696969697,
|
||||
"grad_norm": 1.9233706506478618,
|
||||
"learning_rate": 0.0,
|
||||
"loss": 0.0238,
|
||||
"step": 615
|
||||
},
|
||||
{
|
||||
"epoch": 4.96969696969697,
|
||||
"step": 615,
|
||||
"total_flos": 2730568089600.0,
|
||||
"train_loss": 0.6338221309388556,
|
||||
"train_runtime": 7944.002,
|
||||
"train_samples_per_second": 2.492,
|
||||
"train_steps_per_second": 0.077
|
||||
}
|
||||
],
|
||||
"logging_steps": 5,
|
||||
"max_steps": 615,
|
||||
"num_input_tokens_seen": 0,
|
||||
"num_train_epochs": 5,
|
||||
"save_steps": 500,
|
||||
"stateful_callbacks": {
|
||||
"TrainerControl": {
|
||||
"args": {
|
||||
"should_epoch_stop": false,
|
||||
"should_evaluate": false,
|
||||
"should_log": false,
|
||||
"should_save": false,
|
||||
"should_training_stop": false
|
||||
},
|
||||
"attributes": {}
|
||||
}
|
||||
},
|
||||
"total_flos": 2730568089600.0,
|
||||
"train_batch_size": 4,
|
||||
"trial_name": null,
|
||||
"trial_params": null
|
||||
}
|
||||
3
training_args.bin
Normal file
3
training_args.bin
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:a99978f169115d9f660745efa7701ac2a33d7c63957e2031af135a215e6f4bd8
|
||||
size 6904
|
||||
Reference in New Issue
Block a user