初始化项目，由ModelHub XC社区提供模型
Model: arunasank/xz4e78xm Source: Original Platform
This commit is contained in:
ModelHub XC
4
checkpoint-500/chat_template.jinja
Normal file
4
checkpoint-500/chat_template.jinja
Normal file
@@ -0,0 +1,4 @@
|
||||
{{ bos_token }}{% if messages[0]['role'] == 'system' %}{{ raise_exception('System role not supported') }}{% endif %}{% for message in messages %}{% if (message['role'] == 'user') != (loop.index0 % 2 == 0) %}{{ raise_exception('Conversation roles must alternate user/assistant/user/assistant/...') }}{% endif %}{% if (message['role'] == 'assistant') %}{% set role = 'model' %}{% else %}{% set role = message['role'] %}{% endif %}{{ '<start_of_turn>' + role + '
|
||||
' + message['content'] | trim + '<end_of_turn>
|
||||
' }}{% endfor %}{% if add_generation_prompt %}{{'<start_of_turn>model
|
||||
'}}{% endif %}
|
||||
77
checkpoint-500/config.json
Normal file
77
checkpoint-500/config.json
Normal file
@@ -0,0 +1,77 @@
|
||||
{
|
||||
"architectures": [
|
||||
"Gemma2ForCausalLM"
|
||||
],
|
||||
"attention_bias": false,
|
||||
"attention_dropout": 0.0,
|
||||
"attn_logit_softcapping": 50.0,
|
||||
"bos_token_id": 2,
|
||||
"cache_implementation": "hybrid",
|
||||
"dtype": "bfloat16",
|
||||
"eos_token_id": 1,
|
||||
"final_logit_softcapping": 30.0,
|
||||
"head_dim": 256,
|
||||
"hidden_act": "gelu_pytorch_tanh",
|
||||
"hidden_activation": "gelu_pytorch_tanh",
|
||||
"hidden_size": 3584,
|
||||
"initializer_range": 0.02,
|
||||
"intermediate_size": 14336,
|
||||
"layer_types": [
|
||||
"sliding_attention",
|
||||
"full_attention",
|
||||
"sliding_attention",
|
||||
"full_attention",
|
||||
"sliding_attention",
|
||||
"full_attention",
|
||||
"sliding_attention",
|
||||
"full_attention",
|
||||
"sliding_attention",
|
||||
"full_attention",
|
||||
"sliding_attention",
|
||||
"full_attention",
|
||||
"sliding_attention",
|
||||
"full_attention",
|
||||
"sliding_attention",
|
||||
"full_attention",
|
||||
"sliding_attention",
|
||||
"full_attention",
|
||||
"sliding_attention",
|
||||
"full_attention",
|
||||
"sliding_attention",
|
||||
"full_attention",
|
||||
"sliding_attention",
|
||||
"full_attention",
|
||||
"sliding_attention",
|
||||
"full_attention",
|
||||
"sliding_attention",
|
||||
"full_attention",
|
||||
"sliding_attention",
|
||||
"full_attention",
|
||||
"sliding_attention",
|
||||
"full_attention",
|
||||
"sliding_attention",
|
||||
"full_attention",
|
||||
"sliding_attention",
|
||||
"full_attention",
|
||||
"sliding_attention",
|
||||
"full_attention",
|
||||
"sliding_attention",
|
||||
"full_attention",
|
||||
"sliding_attention",
|
||||
"full_attention"
|
||||
],
|
||||
"max_position_embeddings": 8192,
|
||||
"model_type": "gemma2",
|
||||
"num_attention_heads": 16,
|
||||
"num_hidden_layers": 42,
|
||||
"num_key_value_heads": 8,
|
||||
"pad_token_id": 0,
|
||||
"query_pre_attn_scalar": 256,
|
||||
"rms_norm_eps": 1e-06,
|
||||
"rope_theta": 10000.0,
|
||||
"sliding_window": 4096,
|
||||
"sliding_window_size": 4096,
|
||||
"transformers_version": "4.56.1",
|
||||
"use_cache": true,
|
||||
"vocab_size": 256000
|
||||
}
|
||||
9
checkpoint-500/generation_config.json
Normal file
9
checkpoint-500/generation_config.json
Normal file
@@ -0,0 +1,9 @@
|
||||
{
|
||||
"_from_model_config": true,
|
||||
"bos_token_id": 2,
|
||||
"eos_token_id": [
|
||||
1
|
||||
],
|
||||
"pad_token_id": 0,
|
||||
"transformers_version": "4.56.1"
|
||||
}
|
||||
3
checkpoint-500/model-00001-of-00004.safetensors
Normal file
3
checkpoint-500/model-00001-of-00004.safetensors
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:a0d4eb4fcbddfe01b0dc7a58386a3980826ededaf0ecd67d1ef38995b89615ba
|
||||
size 4903351912
|
||||
3
checkpoint-500/model-00002-of-00004.safetensors
Normal file
3
checkpoint-500/model-00002-of-00004.safetensors
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:712a589c333f49f7b6f3870838a9e0ab4b3c0260abd5130e8c54316e6c99640b
|
||||
size 4947570872
|
||||
3
checkpoint-500/model-00003-of-00004.safetensors
Normal file
3
checkpoint-500/model-00003-of-00004.safetensors
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:9c02d09abebe583ce91b7b324b0fa1b756d5810e64980f7fa42a0a1bfc862cf7
|
||||
size 4962221464
|
||||
3
checkpoint-500/model-00004-of-00004.safetensors
Normal file
3
checkpoint-500/model-00004-of-00004.safetensors
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:cf3d2e24e190e1fc37e2cfc976445af2b9ba48615d1f8c4b58717d920a1f4989
|
||||
size 3670322200
|
||||
472
checkpoint-500/model.safetensors.index.json
Normal file
472
checkpoint-500/model.safetensors.index.json
Normal file
@@ -0,0 +1,472 @@
|
||||
{
|
||||
"metadata": {
|
||||
"total_parameters": 9241705984,
|
||||
"total_size": 18483411968
|
||||
},
|
||||
"weight_map": {
|
||||
"model.embed_tokens.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.0.input_layernorm.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.0.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.0.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.0.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.0.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.0.post_feedforward_layernorm.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.0.pre_feedforward_layernorm.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.0.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.0.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.0.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.0.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.1.input_layernorm.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.1.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.1.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.1.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.1.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.1.post_feedforward_layernorm.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.1.pre_feedforward_layernorm.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.1.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.1.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.1.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.1.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.10.input_layernorm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.10.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.10.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.10.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.10.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.10.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.10.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.10.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.10.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.10.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.10.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.11.input_layernorm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.11.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.11.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.11.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.11.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.11.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.11.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.11.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.11.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.11.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.11.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.12.input_layernorm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.12.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.12.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.12.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.12.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.12.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.12.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.12.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.12.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.12.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.12.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.13.input_layernorm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.13.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.13.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.13.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.13.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.13.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.13.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.13.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.13.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.13.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.13.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.14.input_layernorm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.14.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.14.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.14.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.14.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.14.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.14.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.14.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.14.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.14.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.14.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.15.input_layernorm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.15.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.15.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.15.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.15.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.15.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.15.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.15.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.15.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.15.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.15.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.16.input_layernorm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.16.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.16.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.16.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.16.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.16.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.16.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.16.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.16.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.16.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.16.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.17.input_layernorm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.17.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.17.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.17.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.17.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.17.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.17.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.17.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.17.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.17.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.17.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.18.input_layernorm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.18.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.18.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.18.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.18.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.18.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.18.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.18.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.18.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.18.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.18.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.19.input_layernorm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.19.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.19.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.19.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.19.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.19.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.19.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.19.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.19.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.19.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.19.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.2.input_layernorm.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.2.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.2.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.2.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.2.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.2.post_feedforward_layernorm.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.2.pre_feedforward_layernorm.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.2.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.2.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.2.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.2.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.20.input_layernorm.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.20.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.20.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.20.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.20.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.20.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.20.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.20.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.20.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.20.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.20.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.21.input_layernorm.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.21.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.21.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.21.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.21.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.21.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.21.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.21.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.21.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.21.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.21.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.22.input_layernorm.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.22.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.22.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.22.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.22.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.22.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.22.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.22.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.22.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.22.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.22.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.23.input_layernorm.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.23.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.23.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.23.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.23.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.23.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.23.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.23.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.23.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.23.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.23.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.24.input_layernorm.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.24.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.24.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.24.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.24.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.24.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.24.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.24.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.24.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.24.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.24.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.25.input_layernorm.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.25.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.25.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.25.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.25.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.25.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.25.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.25.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.25.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.25.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.25.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.26.input_layernorm.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.26.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.26.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.26.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.26.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.26.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.26.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.26.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.26.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.26.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.26.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.27.input_layernorm.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.27.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.27.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.27.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.27.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.27.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.27.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.27.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.27.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.27.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.27.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.28.input_layernorm.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.28.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.28.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.28.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.28.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.28.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.28.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.28.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.28.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.28.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.28.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.29.input_layernorm.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.29.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.29.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.29.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.29.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.29.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.29.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.29.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.29.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.29.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.29.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.3.input_layernorm.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.3.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.3.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.3.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.3.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.3.post_feedforward_layernorm.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.3.pre_feedforward_layernorm.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.3.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.3.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.3.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.3.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.30.input_layernorm.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.30.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.30.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.30.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.30.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.30.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.30.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.30.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.30.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.30.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.30.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.31.input_layernorm.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.31.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.31.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.31.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.31.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.31.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.31.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.31.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.31.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.31.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.31.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.32.input_layernorm.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.32.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.32.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.32.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.32.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.32.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.32.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.32.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.32.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.32.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.32.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.33.input_layernorm.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.33.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.33.mlp.gate_proj.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.33.mlp.up_proj.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.33.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.33.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.33.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.33.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.33.self_attn.o_proj.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.33.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.33.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.34.input_layernorm.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.34.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.34.mlp.gate_proj.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.34.mlp.up_proj.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.34.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.34.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.34.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.34.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.34.self_attn.o_proj.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.34.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.34.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.35.input_layernorm.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.35.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.35.mlp.gate_proj.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.35.mlp.up_proj.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.35.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.35.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.35.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.35.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.35.self_attn.o_proj.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.35.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.35.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.36.input_layernorm.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.36.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.36.mlp.gate_proj.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.36.mlp.up_proj.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.36.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.36.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.36.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.36.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.36.self_attn.o_proj.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.36.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.36.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.37.input_layernorm.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.37.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.37.mlp.gate_proj.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.37.mlp.up_proj.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.37.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.37.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.37.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.37.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.37.self_attn.o_proj.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.37.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.37.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.38.input_layernorm.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.38.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.38.mlp.gate_proj.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.38.mlp.up_proj.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.38.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.38.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.38.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.38.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.38.self_attn.o_proj.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.38.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.38.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.39.input_layernorm.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.39.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.39.mlp.gate_proj.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.39.mlp.up_proj.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.39.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.39.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.39.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.39.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.39.self_attn.o_proj.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.39.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.39.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.4.input_layernorm.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.4.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.4.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.4.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.4.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.4.post_feedforward_layernorm.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.4.pre_feedforward_layernorm.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.4.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.4.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.4.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.4.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.40.input_layernorm.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.40.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.40.mlp.gate_proj.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.40.mlp.up_proj.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.40.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.40.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.40.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.40.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.40.self_attn.o_proj.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.40.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.40.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.41.input_layernorm.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.41.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.41.mlp.gate_proj.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.41.mlp.up_proj.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.41.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.41.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.41.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.41.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.41.self_attn.o_proj.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.41.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.41.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.5.input_layernorm.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.5.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.5.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.5.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.5.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.5.post_feedforward_layernorm.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.5.pre_feedforward_layernorm.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.5.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.5.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.5.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.5.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.6.input_layernorm.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.6.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.6.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.6.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.6.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.6.post_feedforward_layernorm.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.6.pre_feedforward_layernorm.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.6.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.6.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.6.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.6.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.7.input_layernorm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.7.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.7.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.7.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.7.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.7.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.7.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.7.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.7.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.7.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.7.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.8.input_layernorm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.8.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.8.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.8.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.8.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.8.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.8.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.8.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.8.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.8.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.8.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.9.input_layernorm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.9.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.9.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.9.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.9.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.9.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.9.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.9.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.9.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.9.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.9.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.norm.weight": "model-00004-of-00004.safetensors"
|
||||
}
|
||||
}
|
||||
3
checkpoint-500/optimizer.pt
Normal file
3
checkpoint-500/optimizer.pt
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:d0e86dd33a09d9589e8b18beb6abd96dd1a50baf221fb85b151165d283486aad
|
||||
size 1585596266
|
||||
3
checkpoint-500/rng_state.pth
Normal file
3
checkpoint-500/rng_state.pth
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:fde986f97ecc270a34049538e5bcb495edbc4e7f5914a76b3cad083de287dc73
|
||||
size 14645
|
||||
3
checkpoint-500/scheduler.pt
Normal file
3
checkpoint-500/scheduler.pt
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:ddc00081d4f892d131a0d2b4d4827a214275362bfa62dfb8413f489c1bfad457
|
||||
size 1465
|
||||
34
checkpoint-500/special_tokens_map.json
Normal file
34
checkpoint-500/special_tokens_map.json
Normal file
@@ -0,0 +1,34 @@
|
||||
{
|
||||
"additional_special_tokens": [
|
||||
"<start_of_turn>",
|
||||
"<end_of_turn>"
|
||||
],
|
||||
"bos_token": {
|
||||
"content": "<bos>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false
|
||||
},
|
||||
"eos_token": {
|
||||
"content": "<eos>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false
|
||||
},
|
||||
"pad_token": {
|
||||
"content": "<pad>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false
|
||||
},
|
||||
"unk_token": {
|
||||
"content": "<unk>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false
|
||||
}
|
||||
}
|
||||
3
checkpoint-500/tokenizer.json
Normal file
3
checkpoint-500/tokenizer.json
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:5f7eee611703c5ce5d1eee32d9cdcfe465647b8aff0c1dfb3bed7ad7dbb05060
|
||||
size 34362873
|
||||
3
checkpoint-500/tokenizer.model
Normal file
3
checkpoint-500/tokenizer.model
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:61a7b147390c64585d6c3543dd6fc636906c9af3865a5548f27f31aee1d4c8e2
|
||||
size 4241003
|
||||
2013
checkpoint-500/tokenizer_config.json
Normal file
2013
checkpoint-500/tokenizer_config.json
Normal file
File diff suppressed because it is too large
Load Diff
754
checkpoint-500/trainer_state.json
Normal file
754
checkpoint-500/trainer_state.json
Normal file
@@ -0,0 +1,754 @@
|
||||
{
|
||||
"best_global_step": null,
|
||||
"best_metric": null,
|
||||
"best_model_checkpoint": null,
|
||||
"epoch": 0.8901746967842439,
|
||||
"eval_steps": 100,
|
||||
"global_step": 500,
|
||||
"is_hyper_param_search": false,
|
||||
"is_local_process_zero": true,
|
||||
"is_world_process_zero": true,
|
||||
"log_history": [
|
||||
{
|
||||
"entropy": 1.3253214355558156,
|
||||
"epoch": 0.017803493935684877,
|
||||
"grad_norm": 14.5,
|
||||
"learning_rate": 6.000000000000001e-07,
|
||||
"loss": 2.1959,
|
||||
"mean_token_accuracy": 0.5975642748177051,
|
||||
"num_tokens": 259048.0,
|
||||
"step": 10
|
||||
},
|
||||
{
|
||||
"entropy": 1.3036176152527332,
|
||||
"epoch": 0.035606987871369754,
|
||||
"grad_norm": 12.75,
|
||||
"learning_rate": 1.2666666666666669e-06,
|
||||
"loss": 2.1499,
|
||||
"mean_token_accuracy": 0.6023047739639878,
|
||||
"num_tokens": 532019.0,
|
||||
"step": 20
|
||||
},
|
||||
{
|
||||
"entropy": 1.3484113823622466,
|
||||
"epoch": 0.053410481807054634,
|
||||
"grad_norm": 10.8125,
|
||||
"learning_rate": 1.9333333333333336e-06,
|
||||
"loss": 2.108,
|
||||
"mean_token_accuracy": 0.6078463492915034,
|
||||
"num_tokens": 807220.0,
|
||||
"step": 30
|
||||
},
|
||||
{
|
||||
"entropy": 1.4311681490391492,
|
||||
"epoch": 0.07121397574273951,
|
||||
"grad_norm": 7.875,
|
||||
"learning_rate": 2.6e-06,
|
||||
"loss": 2.0349,
|
||||
"mean_token_accuracy": 0.6107733339071274,
|
||||
"num_tokens": 1068429.0,
|
||||
"step": 40
|
||||
},
|
||||
{
|
||||
"entropy": 1.479667642340064,
|
||||
"epoch": 0.08901746967842439,
|
||||
"grad_norm": 6.4375,
|
||||
"learning_rate": 3.266666666666667e-06,
|
||||
"loss": 1.8649,
|
||||
"mean_token_accuracy": 0.630409375578165,
|
||||
"num_tokens": 1336698.0,
|
||||
"step": 50
|
||||
},
|
||||
{
|
||||
"entropy": 1.5030938494950532,
|
||||
"epoch": 0.10682096361410927,
|
||||
"grad_norm": 4.1875,
|
||||
"learning_rate": 3.9333333333333335e-06,
|
||||
"loss": 1.7217,
|
||||
"mean_token_accuracy": 0.6490663634613156,
|
||||
"num_tokens": 1608574.0,
|
||||
"step": 60
|
||||
},
|
||||
{
|
||||
"entropy": 1.5066865295171739,
|
||||
"epoch": 0.12462445754979415,
|
||||
"grad_norm": 2.71875,
|
||||
"learning_rate": 4.600000000000001e-06,
|
||||
"loss": 1.5716,
|
||||
"mean_token_accuracy": 0.6711439102888107,
|
||||
"num_tokens": 1872747.0,
|
||||
"step": 70
|
||||
},
|
||||
{
|
||||
"entropy": 1.3747932206839324,
|
||||
"epoch": 0.14242795148547902,
|
||||
"grad_norm": 2.234375,
|
||||
"learning_rate": 5.2666666666666665e-06,
|
||||
"loss": 1.4403,
|
||||
"mean_token_accuracy": 0.6936415441334247,
|
||||
"num_tokens": 2133173.0,
|
||||
"step": 80
|
||||
},
|
||||
{
|
||||
"entropy": 1.1736511185765266,
|
||||
"epoch": 0.1602314454211639,
|
||||
"grad_norm": 1.921875,
|
||||
"learning_rate": 5.933333333333335e-06,
|
||||
"loss": 1.2902,
|
||||
"mean_token_accuracy": 0.7222886722534895,
|
||||
"num_tokens": 2406145.0,
|
||||
"step": 90
|
||||
},
|
||||
{
|
||||
"entropy": 1.1317794900387526,
|
||||
"epoch": 0.17803493935684878,
|
||||
"grad_norm": 1.8046875,
|
||||
"learning_rate": 6.600000000000001e-06,
|
||||
"loss": 1.2424,
|
||||
"mean_token_accuracy": 0.7295983772724867,
|
||||
"num_tokens": 2663709.0,
|
||||
"step": 100
|
||||
},
|
||||
{
|
||||
"epoch": 0.17803493935684878,
|
||||
"eval_biology_entropy": 2.3133783559799195,
|
||||
"eval_biology_loss": 2.7910587787628174,
|
||||
"eval_biology_mean_token_accuracy": 0.5260232911109924,
|
||||
"eval_biology_num_tokens": 2663709.0,
|
||||
"eval_biology_runtime": 39.1656,
|
||||
"eval_biology_samples_per_second": 12.766,
|
||||
"eval_biology_steps_per_second": 3.192,
|
||||
"step": 100
|
||||
},
|
||||
{
|
||||
"epoch": 0.17803493935684878,
|
||||
"eval_chemistry_entropy": 1.0788086318969727,
|
||||
"eval_chemistry_loss": 1.2403336763381958,
|
||||
"eval_chemistry_mean_token_accuracy": 0.7357268543243408,
|
||||
"eval_chemistry_num_tokens": 2663709.0,
|
||||
"eval_chemistry_runtime": 48.7822,
|
||||
"eval_chemistry_samples_per_second": 10.25,
|
||||
"eval_chemistry_steps_per_second": 2.562,
|
||||
"step": 100
|
||||
},
|
||||
{
|
||||
"epoch": 0.17803493935684878,
|
||||
"eval_math_entropy": 0.938195451259613,
|
||||
"eval_math_loss": 1.3248047828674316,
|
||||
"eval_math_mean_token_accuracy": 0.7312402629852295,
|
||||
"eval_math_num_tokens": 2663709.0,
|
||||
"eval_math_runtime": 50.197,
|
||||
"eval_math_samples_per_second": 9.961,
|
||||
"eval_math_steps_per_second": 2.49,
|
||||
"step": 100
|
||||
},
|
||||
{
|
||||
"epoch": 0.17803493935684878,
|
||||
"eval_physics_entropy": 1.022246531009674,
|
||||
"eval_physics_loss": 1.176153302192688,
|
||||
"eval_physics_mean_token_accuracy": 0.7475795021057129,
|
||||
"eval_physics_num_tokens": 2663709.0,
|
||||
"eval_physics_runtime": 57.6895,
|
||||
"eval_physics_samples_per_second": 8.667,
|
||||
"eval_physics_steps_per_second": 2.167,
|
||||
"step": 100
|
||||
},
|
||||
{
|
||||
"entropy": 1.0380497876554728,
|
||||
"epoch": 0.19583843329253367,
|
||||
"grad_norm": 1.7421875,
|
||||
"learning_rate": 7.266666666666668e-06,
|
||||
"loss": 1.157,
|
||||
"mean_token_accuracy": 0.7445634145289659,
|
||||
"num_tokens": 2934980.0,
|
||||
"step": 110
|
||||
},
|
||||
{
|
||||
"entropy": 1.0149445166811346,
|
||||
"epoch": 0.21364192722821854,
|
||||
"grad_norm": 1.6015625,
|
||||
"learning_rate": 7.933333333333334e-06,
|
||||
"loss": 1.1167,
|
||||
"mean_token_accuracy": 0.7485828153789044,
|
||||
"num_tokens": 3208702.0,
|
||||
"step": 120
|
||||
},
|
||||
{
|
||||
"entropy": 0.9639404848217964,
|
||||
"epoch": 0.2314454211639034,
|
||||
"grad_norm": 1.6953125,
|
||||
"learning_rate": 8.6e-06,
|
||||
"loss": 1.0979,
|
||||
"mean_token_accuracy": 0.7553199239075183,
|
||||
"num_tokens": 3480119.0,
|
||||
"step": 130
|
||||
},
|
||||
{
|
||||
"entropy": 0.9687324531376362,
|
||||
"epoch": 0.2492489150995883,
|
||||
"grad_norm": 1.5625,
|
||||
"learning_rate": 9.266666666666667e-06,
|
||||
"loss": 1.0762,
|
||||
"mean_token_accuracy": 0.757096454873681,
|
||||
"num_tokens": 3746392.0,
|
||||
"step": 140
|
||||
},
|
||||
{
|
||||
"entropy": 0.9682734729722142,
|
||||
"epoch": 0.26705240903527316,
|
||||
"grad_norm": 1.875,
|
||||
"learning_rate": 9.933333333333334e-06,
|
||||
"loss": 1.0798,
|
||||
"mean_token_accuracy": 0.7543047085404396,
|
||||
"num_tokens": 4019487.0,
|
||||
"step": 150
|
||||
},
|
||||
{
|
||||
"entropy": 0.9270784268155694,
|
||||
"epoch": 0.28485590297095803,
|
||||
"grad_norm": 1.6015625,
|
||||
"learning_rate": 1.0600000000000002e-05,
|
||||
"loss": 1.047,
|
||||
"mean_token_accuracy": 0.7619738396257162,
|
||||
"num_tokens": 4298100.0,
|
||||
"step": 160
|
||||
},
|
||||
{
|
||||
"entropy": 0.9063327137380839,
|
||||
"epoch": 0.30265939690664295,
|
||||
"grad_norm": 1.5546875,
|
||||
"learning_rate": 1.1266666666666668e-05,
|
||||
"loss": 1.0199,
|
||||
"mean_token_accuracy": 0.7664302848279476,
|
||||
"num_tokens": 4568116.0,
|
||||
"step": 170
|
||||
},
|
||||
{
|
||||
"entropy": 0.9037324048578739,
|
||||
"epoch": 0.3204628908423278,
|
||||
"grad_norm": 1.7109375,
|
||||
"learning_rate": 1.1933333333333335e-05,
|
||||
"loss": 1.0144,
|
||||
"mean_token_accuracy": 0.7657378144562245,
|
||||
"num_tokens": 4831334.0,
|
||||
"step": 180
|
||||
},
|
||||
{
|
||||
"entropy": 0.8886868219822646,
|
||||
"epoch": 0.3382663847780127,
|
||||
"grad_norm": 1.3359375,
|
||||
"learning_rate": 1.2600000000000001e-05,
|
||||
"loss": 0.9874,
|
||||
"mean_token_accuracy": 0.7726409930735827,
|
||||
"num_tokens": 5103301.0,
|
||||
"step": 190
|
||||
},
|
||||
{
|
||||
"entropy": 0.8936551520600915,
|
||||
"epoch": 0.35606987871369755,
|
||||
"grad_norm": 1.3984375,
|
||||
"learning_rate": 1.3266666666666668e-05,
|
||||
"loss": 1.019,
|
||||
"mean_token_accuracy": 0.7648640714585782,
|
||||
"num_tokens": 5362915.0,
|
||||
"step": 200
|
||||
},
|
||||
{
|
||||
"epoch": 0.35606987871369755,
|
||||
"eval_biology_entropy": 2.0171296310424807,
|
||||
"eval_biology_loss": 2.3780677318573,
|
||||
"eval_biology_mean_token_accuracy": 0.5706260974407196,
|
||||
"eval_biology_num_tokens": 5362915.0,
|
||||
"eval_biology_runtime": 39.1203,
|
||||
"eval_biology_samples_per_second": 12.781,
|
||||
"eval_biology_steps_per_second": 3.195,
|
||||
"step": 200
|
||||
},
|
||||
{
|
||||
"epoch": 0.35606987871369755,
|
||||
"eval_chemistry_entropy": 0.8995157589912415,
|
||||
"eval_chemistry_loss": 1.022930383682251,
|
||||
"eval_chemistry_mean_token_accuracy": 0.7701377034187317,
|
||||
"eval_chemistry_num_tokens": 5362915.0,
|
||||
"eval_chemistry_runtime": 48.7263,
|
||||
"eval_chemistry_samples_per_second": 10.261,
|
||||
"eval_chemistry_steps_per_second": 2.565,
|
||||
"step": 200
|
||||
},
|
||||
{
|
||||
"epoch": 0.35606987871369755,
|
||||
"eval_math_entropy": 0.8352043969631195,
|
||||
"eval_math_loss": 1.1696399450302124,
|
||||
"eval_math_mean_token_accuracy": 0.7531440143585205,
|
||||
"eval_math_num_tokens": 5362915.0,
|
||||
"eval_math_runtime": 50.1494,
|
||||
"eval_math_samples_per_second": 9.97,
|
||||
"eval_math_steps_per_second": 2.493,
|
||||
"step": 200
|
||||
},
|
||||
{
|
||||
"epoch": 0.35606987871369755,
|
||||
"eval_physics_entropy": 0.8889413847923279,
|
||||
"eval_physics_loss": 0.9962027072906494,
|
||||
"eval_physics_mean_token_accuracy": 0.775176504611969,
|
||||
"eval_physics_num_tokens": 5362915.0,
|
||||
"eval_physics_runtime": 57.6533,
|
||||
"eval_physics_samples_per_second": 8.673,
|
||||
"eval_physics_steps_per_second": 2.168,
|
||||
"step": 200
|
||||
},
|
||||
{
|
||||
"entropy": 0.8509893301874399,
|
||||
"epoch": 0.3738733726493824,
|
||||
"grad_norm": 1.3359375,
|
||||
"learning_rate": 1.3933333333333334e-05,
|
||||
"loss": 0.9619,
|
||||
"mean_token_accuracy": 0.7772600032389164,
|
||||
"num_tokens": 5625455.0,
|
||||
"step": 210
|
||||
},
|
||||
{
|
||||
"entropy": 0.8560685662552714,
|
||||
"epoch": 0.39167686658506734,
|
||||
"grad_norm": 1.4609375,
|
||||
"learning_rate": 1.46e-05,
|
||||
"loss": 0.9666,
|
||||
"mean_token_accuracy": 0.7764565829187632,
|
||||
"num_tokens": 5890806.0,
|
||||
"step": 220
|
||||
},
|
||||
{
|
||||
"entropy": 0.8282079607248306,
|
||||
"epoch": 0.4094803605207522,
|
||||
"grad_norm": 1.3828125,
|
||||
"learning_rate": 1.5266666666666667e-05,
|
||||
"loss": 0.9353,
|
||||
"mean_token_accuracy": 0.7817027729004622,
|
||||
"num_tokens": 6158751.0,
|
||||
"step": 230
|
||||
},
|
||||
{
|
||||
"entropy": 0.8447791969403624,
|
||||
"epoch": 0.4272838544564371,
|
||||
"grad_norm": 1.4296875,
|
||||
"learning_rate": 1.5933333333333336e-05,
|
||||
"loss": 0.9428,
|
||||
"mean_token_accuracy": 0.7795210376381874,
|
||||
"num_tokens": 6423747.0,
|
||||
"step": 240
|
||||
},
|
||||
{
|
||||
"entropy": 0.8478817906230688,
|
||||
"epoch": 0.44508734839212194,
|
||||
"grad_norm": 1.8984375,
|
||||
"learning_rate": 1.66e-05,
|
||||
"loss": 0.9542,
|
||||
"mean_token_accuracy": 0.7772787544876337,
|
||||
"num_tokens": 6684517.0,
|
||||
"step": 250
|
||||
},
|
||||
{
|
||||
"entropy": 0.8135418241843582,
|
||||
"epoch": 0.4628908423278068,
|
||||
"grad_norm": 1.453125,
|
||||
"learning_rate": 1.726666666666667e-05,
|
||||
"loss": 0.9297,
|
||||
"mean_token_accuracy": 0.783412617444992,
|
||||
"num_tokens": 6945659.0,
|
||||
"step": 260
|
||||
},
|
||||
{
|
||||
"entropy": 0.8251218661665917,
|
||||
"epoch": 0.48069433626349173,
|
||||
"grad_norm": 1.6640625,
|
||||
"learning_rate": 1.7933333333333333e-05,
|
||||
"loss": 0.9307,
|
||||
"mean_token_accuracy": 0.7831020273268223,
|
||||
"num_tokens": 7217797.0,
|
||||
"step": 270
|
||||
},
|
||||
{
|
||||
"entropy": 0.7879339458420873,
|
||||
"epoch": 0.4984978301991766,
|
||||
"grad_norm": 1.328125,
|
||||
"learning_rate": 1.86e-05,
|
||||
"loss": 0.8795,
|
||||
"mean_token_accuracy": 0.7897050529718399,
|
||||
"num_tokens": 7498182.0,
|
||||
"step": 280
|
||||
},
|
||||
{
|
||||
"entropy": 0.8145477209240198,
|
||||
"epoch": 0.5163013241348615,
|
||||
"grad_norm": 1.265625,
|
||||
"learning_rate": 1.926666666666667e-05,
|
||||
"loss": 0.9187,
|
||||
"mean_token_accuracy": 0.7838029216974973,
|
||||
"num_tokens": 7761686.0,
|
||||
"step": 290
|
||||
},
|
||||
{
|
||||
"entropy": 0.8231891892850399,
|
||||
"epoch": 0.5341048180705463,
|
||||
"grad_norm": 1.5625,
|
||||
"learning_rate": 1.9933333333333334e-05,
|
||||
"loss": 0.9271,
|
||||
"mean_token_accuracy": 0.7819279827177524,
|
||||
"num_tokens": 8026923.0,
|
||||
"step": 300
|
||||
},
|
||||
{
|
||||
"epoch": 0.5341048180705463,
|
||||
"eval_biology_entropy": 1.8821040544509888,
|
||||
"eval_biology_loss": 2.226550340652466,
|
||||
"eval_biology_mean_token_accuracy": 0.5873979187011719,
|
||||
"eval_biology_num_tokens": 8026923.0,
|
||||
"eval_biology_runtime": 39.1096,
|
||||
"eval_biology_samples_per_second": 12.785,
|
||||
"eval_biology_steps_per_second": 3.196,
|
||||
"step": 300
|
||||
},
|
||||
{
|
||||
"epoch": 0.5341048180705463,
|
||||
"eval_chemistry_entropy": 0.7922393922805786,
|
||||
"eval_chemistry_loss": 0.9438055157661438,
|
||||
"eval_chemistry_mean_token_accuracy": 0.7841580080986023,
|
||||
"eval_chemistry_num_tokens": 8026923.0,
|
||||
"eval_chemistry_runtime": 48.7268,
|
||||
"eval_chemistry_samples_per_second": 10.261,
|
||||
"eval_chemistry_steps_per_second": 2.565,
|
||||
"step": 300
|
||||
},
|
||||
{
|
||||
"epoch": 0.5341048180705463,
|
||||
"eval_math_entropy": 0.7649447858333588,
|
||||
"eval_math_loss": 1.1145466566085815,
|
||||
"eval_math_mean_token_accuracy": 0.763204463005066,
|
||||
"eval_math_num_tokens": 8026923.0,
|
||||
"eval_math_runtime": 50.1205,
|
||||
"eval_math_samples_per_second": 9.976,
|
||||
"eval_math_steps_per_second": 2.494,
|
||||
"step": 300
|
||||
},
|
||||
{
|
||||
"epoch": 0.5341048180705463,
|
||||
"eval_physics_entropy": 0.7964776389598847,
|
||||
"eval_physics_loss": 0.9373180866241455,
|
||||
"eval_physics_mean_token_accuracy": 0.784841582775116,
|
||||
"eval_physics_num_tokens": 8026923.0,
|
||||
"eval_physics_runtime": 57.6222,
|
||||
"eval_physics_samples_per_second": 8.677,
|
||||
"eval_physics_steps_per_second": 2.169,
|
||||
"step": 300
|
||||
},
|
||||
{
|
||||
"entropy": 0.800599187426269,
|
||||
"epoch": 0.5519083120062312,
|
||||
"grad_norm": 1.6640625,
|
||||
"learning_rate": 1.9933333333333334e-05,
|
||||
"loss": 0.9071,
|
||||
"mean_token_accuracy": 0.7856069568544626,
|
||||
"num_tokens": 8301506.0,
|
||||
"step": 310
|
||||
},
|
||||
{
|
||||
"entropy": 0.7965721761807799,
|
||||
"epoch": 0.5697118059419161,
|
||||
"grad_norm": 1.234375,
|
||||
"learning_rate": 1.985925925925926e-05,
|
||||
"loss": 0.9139,
|
||||
"mean_token_accuracy": 0.786777526140213,
|
||||
"num_tokens": 8559126.0,
|
||||
"step": 320
|
||||
},
|
||||
{
|
||||
"entropy": 0.7925007566809654,
|
||||
"epoch": 0.587515299877601,
|
||||
"grad_norm": 1.4453125,
|
||||
"learning_rate": 1.9785185185185187e-05,
|
||||
"loss": 0.906,
|
||||
"mean_token_accuracy": 0.7864235289394855,
|
||||
"num_tokens": 8821088.0,
|
||||
"step": 330
|
||||
},
|
||||
{
|
||||
"entropy": 0.7728251311928034,
|
||||
"epoch": 0.6053187938132859,
|
||||
"grad_norm": 1.2109375,
|
||||
"learning_rate": 1.971111111111111e-05,
|
||||
"loss": 0.8764,
|
||||
"mean_token_accuracy": 0.7919386122375727,
|
||||
"num_tokens": 9089866.0,
|
||||
"step": 340
|
||||
},
|
||||
{
|
||||
"entropy": 0.7705478269606829,
|
||||
"epoch": 0.6231222877489707,
|
||||
"grad_norm": 1.3515625,
|
||||
"learning_rate": 1.963703703703704e-05,
|
||||
"loss": 0.8837,
|
||||
"mean_token_accuracy": 0.7896027479320764,
|
||||
"num_tokens": 9358866.0,
|
||||
"step": 350
|
||||
},
|
||||
{
|
||||
"entropy": 0.7656249281018972,
|
||||
"epoch": 0.6409257816846556,
|
||||
"grad_norm": 1.1875,
|
||||
"learning_rate": 1.9562962962962964e-05,
|
||||
"loss": 0.8626,
|
||||
"mean_token_accuracy": 0.7931958142668009,
|
||||
"num_tokens": 9629977.0,
|
||||
"step": 360
|
||||
},
|
||||
{
|
||||
"entropy": 0.7598591955378652,
|
||||
"epoch": 0.6587292756203404,
|
||||
"grad_norm": 1.296875,
|
||||
"learning_rate": 1.948888888888889e-05,
|
||||
"loss": 0.8608,
|
||||
"mean_token_accuracy": 0.7945109561085701,
|
||||
"num_tokens": 9897757.0,
|
||||
"step": 370
|
||||
},
|
||||
{
|
||||
"entropy": 0.7398761732503771,
|
||||
"epoch": 0.6765327695560254,
|
||||
"grad_norm": 1.265625,
|
||||
"learning_rate": 1.9414814814814817e-05,
|
||||
"loss": 0.848,
|
||||
"mean_token_accuracy": 0.7985429972410202,
|
||||
"num_tokens": 10168367.0,
|
||||
"step": 380
|
||||
},
|
||||
{
|
||||
"entropy": 0.7331193052232265,
|
||||
"epoch": 0.6943362634917103,
|
||||
"grad_norm": 1.4296875,
|
||||
"learning_rate": 1.9340740740740743e-05,
|
||||
"loss": 0.8366,
|
||||
"mean_token_accuracy": 0.8001827634871006,
|
||||
"num_tokens": 10439039.0,
|
||||
"step": 390
|
||||
},
|
||||
{
|
||||
"entropy": 0.7496613372117281,
|
||||
"epoch": 0.7121397574273951,
|
||||
"grad_norm": 1.15625,
|
||||
"learning_rate": 1.926666666666667e-05,
|
||||
"loss": 0.8517,
|
||||
"mean_token_accuracy": 0.7967161379754544,
|
||||
"num_tokens": 10709549.0,
|
||||
"step": 400
|
||||
},
|
||||
{
|
||||
"epoch": 0.7121397574273951,
|
||||
"eval_biology_entropy": 1.823248610496521,
|
||||
"eval_biology_loss": 2.159681558609009,
|
||||
"eval_biology_mean_token_accuracy": 0.5972214796543122,
|
||||
"eval_biology_num_tokens": 10709549.0,
|
||||
"eval_biology_runtime": 39.2024,
|
||||
"eval_biology_samples_per_second": 12.754,
|
||||
"eval_biology_steps_per_second": 3.189,
|
||||
"step": 400
|
||||
},
|
||||
{
|
||||
"epoch": 0.7121397574273951,
|
||||
"eval_chemistry_entropy": 0.7672218294143677,
|
||||
"eval_chemistry_loss": 0.8971230387687683,
|
||||
"eval_chemistry_mean_token_accuracy": 0.7928389143943787,
|
||||
"eval_chemistry_num_tokens": 10709549.0,
|
||||
"eval_chemistry_runtime": 48.8296,
|
||||
"eval_chemistry_samples_per_second": 10.24,
|
||||
"eval_chemistry_steps_per_second": 2.56,
|
||||
"step": 400
|
||||
},
|
||||
{
|
||||
"epoch": 0.7121397574273951,
|
||||
"eval_math_entropy": 0.7494234163761139,
|
||||
"eval_math_loss": 1.0849554538726807,
|
||||
"eval_math_mean_token_accuracy": 0.7666959943771362,
|
||||
"eval_math_num_tokens": 10709549.0,
|
||||
"eval_math_runtime": 50.2325,
|
||||
"eval_math_samples_per_second": 9.954,
|
||||
"eval_math_steps_per_second": 2.488,
|
||||
"step": 400
|
||||
},
|
||||
{
|
||||
"epoch": 0.7121397574273951,
|
||||
"eval_physics_entropy": 0.7731240901947022,
|
||||
"eval_physics_loss": 0.9018430709838867,
|
||||
"eval_physics_mean_token_accuracy": 0.7909709796905517,
|
||||
"eval_physics_num_tokens": 10709549.0,
|
||||
"eval_physics_runtime": 57.7809,
|
||||
"eval_physics_samples_per_second": 8.653,
|
||||
"eval_physics_steps_per_second": 2.163,
|
||||
"step": 400
|
||||
},
|
||||
{
|
||||
"entropy": 0.7354462265968322,
|
||||
"epoch": 0.72994325136308,
|
||||
"grad_norm": 1.3125,
|
||||
"learning_rate": 1.9192592592592593e-05,
|
||||
"loss": 0.8411,
|
||||
"mean_token_accuracy": 0.8003354988992214,
|
||||
"num_tokens": 10976011.0,
|
||||
"step": 410
|
||||
},
|
||||
{
|
||||
"entropy": 0.7435122076421976,
|
||||
"epoch": 0.7477467452987648,
|
||||
"grad_norm": 1.15625,
|
||||
"learning_rate": 1.911851851851852e-05,
|
||||
"loss": 0.8438,
|
||||
"mean_token_accuracy": 0.7971596572548151,
|
||||
"num_tokens": 11245251.0,
|
||||
"step": 420
|
||||
},
|
||||
{
|
||||
"entropy": 0.759309609234333,
|
||||
"epoch": 0.7655502392344498,
|
||||
"grad_norm": 1.375,
|
||||
"learning_rate": 1.9044444444444446e-05,
|
||||
"loss": 0.8579,
|
||||
"mean_token_accuracy": 0.792990094423294,
|
||||
"num_tokens": 11518514.0,
|
||||
"step": 430
|
||||
},
|
||||
{
|
||||
"entropy": 0.7545185785740613,
|
||||
"epoch": 0.7833537331701347,
|
||||
"grad_norm": 1.3125,
|
||||
"learning_rate": 1.8970370370370372e-05,
|
||||
"loss": 0.8574,
|
||||
"mean_token_accuracy": 0.794682926684618,
|
||||
"num_tokens": 11787117.0,
|
||||
"step": 440
|
||||
},
|
||||
{
|
||||
"entropy": 0.751254404336214,
|
||||
"epoch": 0.8011572271058195,
|
||||
"grad_norm": 1.515625,
|
||||
"learning_rate": 1.8896296296296295e-05,
|
||||
"loss": 0.8495,
|
||||
"mean_token_accuracy": 0.7972703959792853,
|
||||
"num_tokens": 12050993.0,
|
||||
"step": 450
|
||||
},
|
||||
{
|
||||
"entropy": 0.7543802969157696,
|
||||
"epoch": 0.8189607210415044,
|
||||
"grad_norm": 1.28125,
|
||||
"learning_rate": 1.8822222222222225e-05,
|
||||
"loss": 0.8527,
|
||||
"mean_token_accuracy": 0.7960003320127725,
|
||||
"num_tokens": 12311688.0,
|
||||
"step": 460
|
||||
},
|
||||
{
|
||||
"entropy": 0.7186065271496773,
|
||||
"epoch": 0.8367642149771892,
|
||||
"grad_norm": 1.15625,
|
||||
"learning_rate": 1.874814814814815e-05,
|
||||
"loss": 0.8096,
|
||||
"mean_token_accuracy": 0.8027401898056269,
|
||||
"num_tokens": 12598408.0,
|
||||
"step": 470
|
||||
},
|
||||
{
|
||||
"entropy": 0.7257912319153548,
|
||||
"epoch": 0.8545677089128741,
|
||||
"grad_norm": 1.125,
|
||||
"learning_rate": 1.8674074074074075e-05,
|
||||
"loss": 0.8299,
|
||||
"mean_token_accuracy": 0.8012042317539454,
|
||||
"num_tokens": 12869642.0,
|
||||
"step": 480
|
||||
},
|
||||
{
|
||||
"entropy": 0.7173270899802446,
|
||||
"epoch": 0.8723712028485591,
|
||||
"grad_norm": 1.125,
|
||||
"learning_rate": 1.86e-05,
|
||||
"loss": 0.8146,
|
||||
"mean_token_accuracy": 0.8032398864626884,
|
||||
"num_tokens": 13139555.0,
|
||||
"step": 490
|
||||
},
|
||||
{
|
||||
"entropy": 0.7487573977559805,
|
||||
"epoch": 0.8901746967842439,
|
||||
"grad_norm": 1.2421875,
|
||||
"learning_rate": 1.8525925925925928e-05,
|
||||
"loss": 0.8562,
|
||||
"mean_token_accuracy": 0.7955521304160357,
|
||||
"num_tokens": 13396545.0,
|
||||
"step": 500
|
||||
},
|
||||
{
|
||||
"epoch": 0.8901746967842439,
|
||||
"eval_biology_entropy": 1.8204833917617798,
|
||||
"eval_biology_loss": 2.1014022827148438,
|
||||
"eval_biology_mean_token_accuracy": 0.6051526765823364,
|
||||
"eval_biology_num_tokens": 13396545.0,
|
||||
"eval_biology_runtime": 39.1543,
|
||||
"eval_biology_samples_per_second": 12.77,
|
||||
"eval_biology_steps_per_second": 3.192,
|
||||
"step": 500
|
||||
},
|
||||
{
|
||||
"epoch": 0.8901746967842439,
|
||||
"eval_chemistry_entropy": 0.7538458592891694,
|
||||
"eval_chemistry_loss": 0.8699584007263184,
|
||||
"eval_chemistry_mean_token_accuracy": 0.7975720787048339,
|
||||
"eval_chemistry_num_tokens": 13396545.0,
|
||||
"eval_chemistry_runtime": 48.8229,
|
||||
"eval_chemistry_samples_per_second": 10.241,
|
||||
"eval_chemistry_steps_per_second": 2.56,
|
||||
"step": 500
|
||||
},
|
||||
{
|
||||
"epoch": 0.8901746967842439,
|
||||
"eval_math_entropy": 0.7398368701934814,
|
||||
"eval_math_loss": 1.0758970975875854,
|
||||
"eval_math_mean_token_accuracy": 0.7687290472984314,
|
||||
"eval_math_num_tokens": 13396545.0,
|
||||
"eval_math_runtime": 50.0916,
|
||||
"eval_math_samples_per_second": 9.982,
|
||||
"eval_math_steps_per_second": 2.495,
|
||||
"step": 500
|
||||
},
|
||||
{
|
||||
"epoch": 0.8901746967842439,
|
||||
"eval_physics_entropy": 0.7607374501228332,
|
||||
"eval_physics_loss": 0.8848790526390076,
|
||||
"eval_physics_mean_token_accuracy": 0.7937801394462586,
|
||||
"eval_physics_num_tokens": 13396545.0,
|
||||
"eval_physics_runtime": 57.6599,
|
||||
"eval_physics_samples_per_second": 8.672,
|
||||
"eval_physics_steps_per_second": 2.168,
|
||||
"step": 500
|
||||
}
|
||||
],
|
||||
"logging_steps": 10,
|
||||
"max_steps": 3000,
|
||||
"num_input_tokens_seen": 0,
|
||||
"num_train_epochs": 6,
|
||||
"save_steps": 500,
|
||||
"stateful_callbacks": {
|
||||
"TrainerControl": {
|
||||
"args": {
|
||||
"should_epoch_stop": false,
|
||||
"should_evaluate": false,
|
||||
"should_log": false,
|
||||
"should_save": true,
|
||||
"should_training_stop": false
|
||||
},
|
||||
"attributes": {}
|
||||
}
|
||||
},
|
||||
"total_flos": 1.0399995579963064e+18,
|
||||
"train_batch_size": 4,
|
||||
"trial_name": null,
|
||||
"trial_params": null
|
||||
}
|
||||
3
checkpoint-500/training_args.bin
Normal file
3
checkpoint-500/training_args.bin
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:dfc23fa1b9be77c605cba9733debba5f60170f0eadaa2a8866b5aaf4b0e65756
|
||||
size 6417
|
||||
Reference in New Issue
Block a user