初始化项目，由ModelHub XC社区提供模型
Model: roonbug/jj75i299 Source: Original Platform
This commit is contained in:
ModelHub XC
4
checkpoint-500/chat_template.jinja
Normal file
4
checkpoint-500/chat_template.jinja
Normal file
@@ -0,0 +1,4 @@
|
||||
{{ bos_token }}{% if messages[0]['role'] == 'system' %}{{ raise_exception('System role not supported') }}{% endif %}{% for message in messages %}{% if (message['role'] == 'user') != (loop.index0 % 2 == 0) %}{{ raise_exception('Conversation roles must alternate user/assistant/user/assistant/...') }}{% endif %}{% if (message['role'] == 'assistant') %}{% set role = 'model' %}{% else %}{% set role = message['role'] %}{% endif %}{{ '<start_of_turn>' + role + '
|
||||
' + message['content'] | trim + '<end_of_turn>
|
||||
' }}{% endfor %}{% if add_generation_prompt %}{{'<start_of_turn>model
|
||||
'}}{% endif %}
|
||||
77
checkpoint-500/config.json
Normal file
77
checkpoint-500/config.json
Normal file
@@ -0,0 +1,77 @@
|
||||
{
|
||||
"architectures": [
|
||||
"Gemma2ForCausalLM"
|
||||
],
|
||||
"attention_bias": false,
|
||||
"attention_dropout": 0.0,
|
||||
"attn_logit_softcapping": 50.0,
|
||||
"bos_token_id": 2,
|
||||
"cache_implementation": "hybrid",
|
||||
"dtype": "bfloat16",
|
||||
"eos_token_id": 1,
|
||||
"final_logit_softcapping": 30.0,
|
||||
"head_dim": 256,
|
||||
"hidden_act": "gelu_pytorch_tanh",
|
||||
"hidden_activation": "gelu_pytorch_tanh",
|
||||
"hidden_size": 3584,
|
||||
"initializer_range": 0.02,
|
||||
"intermediate_size": 14336,
|
||||
"layer_types": [
|
||||
"sliding_attention",
|
||||
"full_attention",
|
||||
"sliding_attention",
|
||||
"full_attention",
|
||||
"sliding_attention",
|
||||
"full_attention",
|
||||
"sliding_attention",
|
||||
"full_attention",
|
||||
"sliding_attention",
|
||||
"full_attention",
|
||||
"sliding_attention",
|
||||
"full_attention",
|
||||
"sliding_attention",
|
||||
"full_attention",
|
||||
"sliding_attention",
|
||||
"full_attention",
|
||||
"sliding_attention",
|
||||
"full_attention",
|
||||
"sliding_attention",
|
||||
"full_attention",
|
||||
"sliding_attention",
|
||||
"full_attention",
|
||||
"sliding_attention",
|
||||
"full_attention",
|
||||
"sliding_attention",
|
||||
"full_attention",
|
||||
"sliding_attention",
|
||||
"full_attention",
|
||||
"sliding_attention",
|
||||
"full_attention",
|
||||
"sliding_attention",
|
||||
"full_attention",
|
||||
"sliding_attention",
|
||||
"full_attention",
|
||||
"sliding_attention",
|
||||
"full_attention",
|
||||
"sliding_attention",
|
||||
"full_attention",
|
||||
"sliding_attention",
|
||||
"full_attention",
|
||||
"sliding_attention",
|
||||
"full_attention"
|
||||
],
|
||||
"max_position_embeddings": 8192,
|
||||
"model_type": "gemma2",
|
||||
"num_attention_heads": 16,
|
||||
"num_hidden_layers": 42,
|
||||
"num_key_value_heads": 8,
|
||||
"pad_token_id": 0,
|
||||
"query_pre_attn_scalar": 256,
|
||||
"rms_norm_eps": 1e-06,
|
||||
"rope_theta": 10000.0,
|
||||
"sliding_window": 4096,
|
||||
"sliding_window_size": 4096,
|
||||
"transformers_version": "4.56.1",
|
||||
"use_cache": true,
|
||||
"vocab_size": 256000
|
||||
}
|
||||
9
checkpoint-500/generation_config.json
Normal file
9
checkpoint-500/generation_config.json
Normal file
@@ -0,0 +1,9 @@
|
||||
{
|
||||
"_from_model_config": true,
|
||||
"bos_token_id": 2,
|
||||
"eos_token_id": [
|
||||
1
|
||||
],
|
||||
"pad_token_id": 0,
|
||||
"transformers_version": "4.56.1"
|
||||
}
|
||||
3
checkpoint-500/model-00001-of-00004.safetensors
Normal file
3
checkpoint-500/model-00001-of-00004.safetensors
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:a0d4eb4fcbddfe01b0dc7a58386a3980826ededaf0ecd67d1ef38995b89615ba
|
||||
size 4903351912
|
||||
3
checkpoint-500/model-00002-of-00004.safetensors
Normal file
3
checkpoint-500/model-00002-of-00004.safetensors
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:712a589c333f49f7b6f3870838a9e0ab4b3c0260abd5130e8c54316e6c99640b
|
||||
size 4947570872
|
||||
3
checkpoint-500/model-00003-of-00004.safetensors
Normal file
3
checkpoint-500/model-00003-of-00004.safetensors
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:56a2b3de2fb815156e39befade232902892224592e366bac54e7c67c14172dad
|
||||
size 4962221464
|
||||
3
checkpoint-500/model-00004-of-00004.safetensors
Normal file
3
checkpoint-500/model-00004-of-00004.safetensors
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:fb5fc3d9202aedc4a09bb2bbd63b3248fcafc11d07fc4215099f9b8fa46ef502
|
||||
size 3670322200
|
||||
472
checkpoint-500/model.safetensors.index.json
Normal file
472
checkpoint-500/model.safetensors.index.json
Normal file
@@ -0,0 +1,472 @@
|
||||
{
|
||||
"metadata": {
|
||||
"total_parameters": 9241705984,
|
||||
"total_size": 18483411968
|
||||
},
|
||||
"weight_map": {
|
||||
"model.embed_tokens.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.0.input_layernorm.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.0.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.0.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.0.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.0.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.0.post_feedforward_layernorm.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.0.pre_feedforward_layernorm.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.0.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.0.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.0.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.0.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.1.input_layernorm.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.1.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.1.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.1.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.1.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.1.post_feedforward_layernorm.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.1.pre_feedforward_layernorm.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.1.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.1.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.1.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.1.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.10.input_layernorm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.10.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.10.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.10.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.10.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.10.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.10.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.10.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.10.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.10.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.10.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.11.input_layernorm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.11.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.11.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.11.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.11.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.11.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.11.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.11.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.11.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.11.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.11.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.12.input_layernorm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.12.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.12.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.12.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.12.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.12.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.12.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.12.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.12.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.12.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.12.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.13.input_layernorm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.13.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.13.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.13.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.13.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.13.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.13.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.13.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.13.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.13.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.13.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.14.input_layernorm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.14.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.14.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.14.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.14.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.14.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.14.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.14.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.14.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.14.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.14.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.15.input_layernorm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.15.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.15.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.15.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.15.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.15.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.15.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.15.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.15.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.15.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.15.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.16.input_layernorm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.16.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.16.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.16.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.16.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.16.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.16.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.16.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.16.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.16.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.16.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.17.input_layernorm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.17.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.17.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.17.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.17.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.17.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.17.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.17.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.17.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.17.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.17.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.18.input_layernorm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.18.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.18.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.18.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.18.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.18.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.18.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.18.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.18.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.18.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.18.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.19.input_layernorm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.19.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.19.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.19.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.19.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.19.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.19.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.19.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.19.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.19.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.19.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.2.input_layernorm.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.2.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.2.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.2.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.2.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.2.post_feedforward_layernorm.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.2.pre_feedforward_layernorm.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.2.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.2.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.2.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.2.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.20.input_layernorm.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.20.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.20.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.20.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.20.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.20.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.20.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.20.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.20.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.20.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.20.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.21.input_layernorm.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.21.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.21.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.21.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.21.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.21.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.21.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.21.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.21.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.21.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.21.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.22.input_layernorm.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.22.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.22.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.22.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.22.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.22.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.22.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.22.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.22.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.22.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.22.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.23.input_layernorm.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.23.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.23.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.23.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.23.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.23.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.23.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.23.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.23.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.23.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.23.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.24.input_layernorm.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.24.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.24.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.24.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.24.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.24.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.24.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.24.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.24.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.24.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.24.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.25.input_layernorm.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.25.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.25.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.25.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.25.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.25.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.25.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.25.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.25.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.25.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.25.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.26.input_layernorm.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.26.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.26.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.26.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.26.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.26.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.26.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.26.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.26.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.26.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.26.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.27.input_layernorm.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.27.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.27.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.27.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.27.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.27.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.27.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.27.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.27.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.27.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.27.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.28.input_layernorm.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.28.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.28.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.28.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.28.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.28.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.28.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.28.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.28.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.28.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.28.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.29.input_layernorm.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.29.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.29.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.29.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.29.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.29.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.29.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.29.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.29.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.29.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.29.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.3.input_layernorm.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.3.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.3.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.3.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.3.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.3.post_feedforward_layernorm.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.3.pre_feedforward_layernorm.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.3.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.3.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.3.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.3.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.30.input_layernorm.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.30.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.30.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.30.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.30.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.30.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.30.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.30.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.30.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.30.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.30.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.31.input_layernorm.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.31.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.31.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.31.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.31.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.31.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.31.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.31.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.31.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.31.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.31.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.32.input_layernorm.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.32.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.32.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.32.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.32.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.32.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.32.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.32.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.32.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.32.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.32.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.33.input_layernorm.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.33.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.33.mlp.gate_proj.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.33.mlp.up_proj.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.33.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.33.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.33.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.33.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.33.self_attn.o_proj.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.33.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.33.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.34.input_layernorm.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.34.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.34.mlp.gate_proj.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.34.mlp.up_proj.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.34.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.34.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.34.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.34.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.34.self_attn.o_proj.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.34.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.34.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.35.input_layernorm.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.35.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.35.mlp.gate_proj.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.35.mlp.up_proj.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.35.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.35.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.35.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.35.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.35.self_attn.o_proj.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.35.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.35.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.36.input_layernorm.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.36.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.36.mlp.gate_proj.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.36.mlp.up_proj.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.36.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.36.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.36.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.36.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.36.self_attn.o_proj.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.36.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.36.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.37.input_layernorm.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.37.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.37.mlp.gate_proj.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.37.mlp.up_proj.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.37.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.37.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.37.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.37.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.37.self_attn.o_proj.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.37.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.37.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.38.input_layernorm.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.38.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.38.mlp.gate_proj.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.38.mlp.up_proj.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.38.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.38.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.38.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.38.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.38.self_attn.o_proj.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.38.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.38.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.39.input_layernorm.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.39.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.39.mlp.gate_proj.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.39.mlp.up_proj.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.39.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.39.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.39.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.39.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.39.self_attn.o_proj.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.39.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.39.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.4.input_layernorm.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.4.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.4.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.4.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.4.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.4.post_feedforward_layernorm.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.4.pre_feedforward_layernorm.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.4.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.4.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.4.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.4.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.40.input_layernorm.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.40.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.40.mlp.gate_proj.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.40.mlp.up_proj.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.40.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.40.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.40.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.40.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.40.self_attn.o_proj.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.40.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.40.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.41.input_layernorm.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.41.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.41.mlp.gate_proj.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.41.mlp.up_proj.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.41.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.41.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.41.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.41.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.41.self_attn.o_proj.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.41.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.41.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.5.input_layernorm.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.5.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.5.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.5.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.5.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.5.post_feedforward_layernorm.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.5.pre_feedforward_layernorm.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.5.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.5.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.5.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.5.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.6.input_layernorm.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.6.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.6.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.6.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.6.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.6.post_feedforward_layernorm.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.6.pre_feedforward_layernorm.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.6.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.6.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.6.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.6.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.7.input_layernorm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.7.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.7.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.7.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.7.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.7.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.7.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.7.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.7.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.7.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.7.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.8.input_layernorm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.8.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.8.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.8.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.8.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.8.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.8.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.8.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.8.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.8.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.8.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.9.input_layernorm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.9.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.9.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.9.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.9.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.9.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.9.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.9.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.9.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.9.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.9.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.norm.weight": "model-00004-of-00004.safetensors"
|
||||
}
|
||||
}
|
||||
3
checkpoint-500/optimizer.pt
Normal file
3
checkpoint-500/optimizer.pt
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:94f019da9d99e1adb057f5cc63700cb81e2519a5c1f8382ee2479bb837803d56
|
||||
size 7927918951
|
||||
3
checkpoint-500/rng_state.pth
Normal file
3
checkpoint-500/rng_state.pth
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:69b6509f8bbdd943c9f8984cdc19d5d17ca63b6e47cd6e28080ebf502da06ddc
|
||||
size 14645
|
||||
3
checkpoint-500/scheduler.pt
Normal file
3
checkpoint-500/scheduler.pt
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:63481e620bd287eefb582d20daf306c54c7268088fea6f19af8100945c7e3e7e
|
||||
size 1465
|
||||
34
checkpoint-500/special_tokens_map.json
Normal file
34
checkpoint-500/special_tokens_map.json
Normal file
@@ -0,0 +1,34 @@
|
||||
{
|
||||
"additional_special_tokens": [
|
||||
"<start_of_turn>",
|
||||
"<end_of_turn>"
|
||||
],
|
||||
"bos_token": {
|
||||
"content": "<bos>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false
|
||||
},
|
||||
"eos_token": {
|
||||
"content": "<eos>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false
|
||||
},
|
||||
"pad_token": {
|
||||
"content": "<pad>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false
|
||||
},
|
||||
"unk_token": {
|
||||
"content": "<unk>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false
|
||||
}
|
||||
}
|
||||
3
checkpoint-500/tokenizer.json
Normal file
3
checkpoint-500/tokenizer.json
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:5f7eee611703c5ce5d1eee32d9cdcfe465647b8aff0c1dfb3bed7ad7dbb05060
|
||||
size 34362873
|
||||
3
checkpoint-500/tokenizer.model
Normal file
3
checkpoint-500/tokenizer.model
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:61a7b147390c64585d6c3543dd6fc636906c9af3865a5548f27f31aee1d4c8e2
|
||||
size 4241003
|
||||
2013
checkpoint-500/tokenizer_config.json
Normal file
2013
checkpoint-500/tokenizer_config.json
Normal file
File diff suppressed because it is too large
Load Diff
644
checkpoint-500/trainer_state.json
Normal file
644
checkpoint-500/trainer_state.json
Normal file
@@ -0,0 +1,644 @@
|
||||
{
|
||||
"best_global_step": null,
|
||||
"best_metric": null,
|
||||
"best_model_checkpoint": null,
|
||||
"epoch": 0.8,
|
||||
"eval_steps": 100,
|
||||
"global_step": 500,
|
||||
"is_hyper_param_search": false,
|
||||
"is_local_process_zero": true,
|
||||
"is_world_process_zero": true,
|
||||
"log_history": [
|
||||
{
|
||||
"entropy": 0.6421742392703891,
|
||||
"epoch": 0.016,
|
||||
"grad_norm": 4.4375,
|
||||
"learning_rate": 1.8e-07,
|
||||
"loss": 0.953,
|
||||
"mean_token_accuracy": 0.792544937506318,
|
||||
"num_tokens": 282849.0,
|
||||
"step": 10
|
||||
},
|
||||
{
|
||||
"entropy": 0.6719903893768787,
|
||||
"epoch": 0.032,
|
||||
"grad_norm": 4.25,
|
||||
"learning_rate": 3.8e-07,
|
||||
"loss": 0.9758,
|
||||
"mean_token_accuracy": 0.7857426963746548,
|
||||
"num_tokens": 561726.0,
|
||||
"step": 20
|
||||
},
|
||||
{
|
||||
"entropy": 0.6603780068457127,
|
||||
"epoch": 0.048,
|
||||
"grad_norm": 4.40625,
|
||||
"learning_rate": 5.800000000000001e-07,
|
||||
"loss": 0.9582,
|
||||
"mean_token_accuracy": 0.7894285101443529,
|
||||
"num_tokens": 842092.0,
|
||||
"step": 30
|
||||
},
|
||||
{
|
||||
"entropy": 0.6775440879166126,
|
||||
"epoch": 0.064,
|
||||
"grad_norm": 3.890625,
|
||||
"learning_rate": 7.8e-07,
|
||||
"loss": 0.9776,
|
||||
"mean_token_accuracy": 0.785004175081849,
|
||||
"num_tokens": 1133349.0,
|
||||
"step": 40
|
||||
},
|
||||
{
|
||||
"entropy": 0.6926743468269706,
|
||||
"epoch": 0.08,
|
||||
"grad_norm": 4.6875,
|
||||
"learning_rate": 9.800000000000001e-07,
|
||||
"loss": 0.9497,
|
||||
"mean_token_accuracy": 0.7896979171782732,
|
||||
"num_tokens": 1422340.0,
|
||||
"step": 50
|
||||
},
|
||||
{
|
||||
"entropy": 0.7043748194351792,
|
||||
"epoch": 0.096,
|
||||
"grad_norm": 3.484375,
|
||||
"learning_rate": 1.1800000000000001e-06,
|
||||
"loss": 0.9217,
|
||||
"mean_token_accuracy": 0.7949878334999084,
|
||||
"num_tokens": 1711821.0,
|
||||
"step": 60
|
||||
},
|
||||
{
|
||||
"entropy": 0.7551965553313493,
|
||||
"epoch": 0.112,
|
||||
"grad_norm": 2.71875,
|
||||
"learning_rate": 1.3800000000000001e-06,
|
||||
"loss": 0.931,
|
||||
"mean_token_accuracy": 0.7912098452448845,
|
||||
"num_tokens": 1991855.0,
|
||||
"step": 70
|
||||
},
|
||||
{
|
||||
"entropy": 0.7913430579006672,
|
||||
"epoch": 0.128,
|
||||
"grad_norm": 2.359375,
|
||||
"learning_rate": 1.5800000000000001e-06,
|
||||
"loss": 0.9172,
|
||||
"mean_token_accuracy": 0.7935858864337206,
|
||||
"num_tokens": 2288720.0,
|
||||
"step": 80
|
||||
},
|
||||
{
|
||||
"entropy": 0.7881425807252527,
|
||||
"epoch": 0.144,
|
||||
"grad_norm": 2.125,
|
||||
"learning_rate": 1.7800000000000001e-06,
|
||||
"loss": 0.917,
|
||||
"mean_token_accuracy": 0.7932404045015573,
|
||||
"num_tokens": 2564352.0,
|
||||
"step": 90
|
||||
},
|
||||
{
|
||||
"entropy": 0.7627748234197498,
|
||||
"epoch": 0.16,
|
||||
"grad_norm": 2.125,
|
||||
"learning_rate": 1.98e-06,
|
||||
"loss": 0.9014,
|
||||
"mean_token_accuracy": 0.7970093633979559,
|
||||
"num_tokens": 2841876.0,
|
||||
"step": 100
|
||||
},
|
||||
{
|
||||
"epoch": 0.16,
|
||||
"eval_math_entropy": 0.7972113332748413,
|
||||
"eval_math_loss": 0.9417253136634827,
|
||||
"eval_math_mean_token_accuracy": 0.7907100429534912,
|
||||
"eval_math_num_tokens": 2841876.0,
|
||||
"eval_math_runtime": 35.8728,
|
||||
"eval_math_samples_per_second": 13.938,
|
||||
"eval_math_steps_per_second": 3.485,
|
||||
"step": 100
|
||||
},
|
||||
{
|
||||
"epoch": 0.16,
|
||||
"eval_physics_entropy": 0.7183168392181396,
|
||||
"eval_physics_loss": 0.7405859231948853,
|
||||
"eval_physics_mean_token_accuracy": 0.822186062335968,
|
||||
"eval_physics_num_tokens": 2841876.0,
|
||||
"eval_physics_runtime": 41.4814,
|
||||
"eval_physics_samples_per_second": 12.054,
|
||||
"eval_physics_steps_per_second": 3.013,
|
||||
"step": 100
|
||||
},
|
||||
{
|
||||
"entropy": 0.7770459342747926,
|
||||
"epoch": 0.176,
|
||||
"grad_norm": 1.9765625,
|
||||
"learning_rate": 2.1800000000000003e-06,
|
||||
"loss": 0.8952,
|
||||
"mean_token_accuracy": 0.7969978783279658,
|
||||
"num_tokens": 3129237.0,
|
||||
"step": 110
|
||||
},
|
||||
{
|
||||
"entropy": 0.7958783155307174,
|
||||
"epoch": 0.192,
|
||||
"grad_norm": 2.3125,
|
||||
"learning_rate": 2.38e-06,
|
||||
"loss": 0.9119,
|
||||
"mean_token_accuracy": 0.793396720290184,
|
||||
"num_tokens": 3414875.0,
|
||||
"step": 120
|
||||
},
|
||||
{
|
||||
"entropy": 0.7911858700215817,
|
||||
"epoch": 0.208,
|
||||
"grad_norm": 2.4375,
|
||||
"learning_rate": 2.5800000000000003e-06,
|
||||
"loss": 0.8979,
|
||||
"mean_token_accuracy": 0.7956417176872492,
|
||||
"num_tokens": 3695623.0,
|
||||
"step": 130
|
||||
},
|
||||
{
|
||||
"entropy": 0.7871318949386478,
|
||||
"epoch": 0.224,
|
||||
"grad_norm": 2.09375,
|
||||
"learning_rate": 2.7800000000000005e-06,
|
||||
"loss": 0.8882,
|
||||
"mean_token_accuracy": 0.7958744909614325,
|
||||
"num_tokens": 3977666.0,
|
||||
"step": 140
|
||||
},
|
||||
{
|
||||
"entropy": 0.7629930060356855,
|
||||
"epoch": 0.24,
|
||||
"grad_norm": 1.9453125,
|
||||
"learning_rate": 2.9800000000000003e-06,
|
||||
"loss": 0.8654,
|
||||
"mean_token_accuracy": 0.8001766365021468,
|
||||
"num_tokens": 4269287.0,
|
||||
"step": 150
|
||||
},
|
||||
{
|
||||
"entropy": 0.7563591374084353,
|
||||
"epoch": 0.256,
|
||||
"grad_norm": 2.015625,
|
||||
"learning_rate": 3.1800000000000005e-06,
|
||||
"loss": 0.846,
|
||||
"mean_token_accuracy": 0.8012917254120111,
|
||||
"num_tokens": 4558632.0,
|
||||
"step": 160
|
||||
},
|
||||
{
|
||||
"entropy": 0.7706880155950785,
|
||||
"epoch": 0.272,
|
||||
"grad_norm": 1.859375,
|
||||
"learning_rate": 3.3800000000000007e-06,
|
||||
"loss": 0.8589,
|
||||
"mean_token_accuracy": 0.8001386649906636,
|
||||
"num_tokens": 4839270.0,
|
||||
"step": 170
|
||||
},
|
||||
{
|
||||
"entropy": 0.7551738245412707,
|
||||
"epoch": 0.288,
|
||||
"grad_norm": 1.609375,
|
||||
"learning_rate": 3.58e-06,
|
||||
"loss": 0.8468,
|
||||
"mean_token_accuracy": 0.8015156831592322,
|
||||
"num_tokens": 5125447.0,
|
||||
"step": 180
|
||||
},
|
||||
{
|
||||
"entropy": 0.7806578669697046,
|
||||
"epoch": 0.304,
|
||||
"grad_norm": 1.7734375,
|
||||
"learning_rate": 3.7800000000000002e-06,
|
||||
"loss": 0.8405,
|
||||
"mean_token_accuracy": 0.7998461078852415,
|
||||
"num_tokens": 5415925.0,
|
||||
"step": 190
|
||||
},
|
||||
{
|
||||
"entropy": 0.7316340124234557,
|
||||
"epoch": 0.32,
|
||||
"grad_norm": 1.6953125,
|
||||
"learning_rate": 3.980000000000001e-06,
|
||||
"loss": 0.7942,
|
||||
"mean_token_accuracy": 0.8076768793165684,
|
||||
"num_tokens": 5710178.0,
|
||||
"step": 200
|
||||
},
|
||||
{
|
||||
"epoch": 0.32,
|
||||
"eval_math_entropy": 0.7726770024299622,
|
||||
"eval_math_loss": 0.84871506690979,
|
||||
"eval_math_mean_token_accuracy": 0.7979933972358704,
|
||||
"eval_math_num_tokens": 5710178.0,
|
||||
"eval_math_runtime": 35.765,
|
||||
"eval_math_samples_per_second": 13.98,
|
||||
"eval_math_steps_per_second": 3.495,
|
||||
"step": 200
|
||||
},
|
||||
{
|
||||
"epoch": 0.32,
|
||||
"eval_physics_entropy": 0.6965276994705201,
|
||||
"eval_physics_loss": 0.7036627531051636,
|
||||
"eval_physics_mean_token_accuracy": 0.820941798210144,
|
||||
"eval_physics_num_tokens": 5710178.0,
|
||||
"eval_physics_runtime": 41.4597,
|
||||
"eval_physics_samples_per_second": 12.06,
|
||||
"eval_physics_steps_per_second": 3.015,
|
||||
"step": 200
|
||||
},
|
||||
{
|
||||
"entropy": 0.7737893080338836,
|
||||
"epoch": 0.336,
|
||||
"grad_norm": 1.828125,
|
||||
"learning_rate": 4.18e-06,
|
||||
"loss": 0.8126,
|
||||
"mean_token_accuracy": 0.800426610186696,
|
||||
"num_tokens": 5993843.0,
|
||||
"step": 210
|
||||
},
|
||||
{
|
||||
"entropy": 0.7482354704290628,
|
||||
"epoch": 0.352,
|
||||
"grad_norm": 1.9609375,
|
||||
"learning_rate": 4.38e-06,
|
||||
"loss": 0.7874,
|
||||
"mean_token_accuracy": 0.8061643201857805,
|
||||
"num_tokens": 6279137.0,
|
||||
"step": 220
|
||||
},
|
||||
{
|
||||
"entropy": 0.7608217550441623,
|
||||
"epoch": 0.368,
|
||||
"grad_norm": 1.65625,
|
||||
"learning_rate": 4.58e-06,
|
||||
"loss": 0.7742,
|
||||
"mean_token_accuracy": 0.803000221401453,
|
||||
"num_tokens": 6562387.0,
|
||||
"step": 230
|
||||
},
|
||||
{
|
||||
"entropy": 0.748550821095705,
|
||||
"epoch": 0.384,
|
||||
"grad_norm": 1.84375,
|
||||
"learning_rate": 4.78e-06,
|
||||
"loss": 0.7565,
|
||||
"mean_token_accuracy": 0.8081529751420021,
|
||||
"num_tokens": 6851520.0,
|
||||
"step": 240
|
||||
},
|
||||
{
|
||||
"entropy": 0.7808488950133323,
|
||||
"epoch": 0.4,
|
||||
"grad_norm": 1.75,
|
||||
"learning_rate": 4.980000000000001e-06,
|
||||
"loss": 0.7953,
|
||||
"mean_token_accuracy": 0.7999043371528387,
|
||||
"num_tokens": 7138932.0,
|
||||
"step": 250
|
||||
},
|
||||
{
|
||||
"entropy": 0.7674930268898607,
|
||||
"epoch": 0.416,
|
||||
"grad_norm": 1.6796875,
|
||||
"learning_rate": 5.18e-06,
|
||||
"loss": 0.7597,
|
||||
"mean_token_accuracy": 0.8044768057763576,
|
||||
"num_tokens": 7420720.0,
|
||||
"step": 260
|
||||
},
|
||||
{
|
||||
"entropy": 0.7496220650151372,
|
||||
"epoch": 0.432,
|
||||
"grad_norm": 1.5390625,
|
||||
"learning_rate": 5.380000000000001e-06,
|
||||
"loss": 0.7468,
|
||||
"mean_token_accuracy": 0.8073872365057468,
|
||||
"num_tokens": 7695518.0,
|
||||
"step": 270
|
||||
},
|
||||
{
|
||||
"entropy": 0.7439631385728717,
|
||||
"epoch": 0.448,
|
||||
"grad_norm": 2.21875,
|
||||
"learning_rate": 5.580000000000001e-06,
|
||||
"loss": 0.7339,
|
||||
"mean_token_accuracy": 0.8076559219509363,
|
||||
"num_tokens": 7983543.0,
|
||||
"step": 280
|
||||
},
|
||||
{
|
||||
"entropy": 0.7457249015569687,
|
||||
"epoch": 0.464,
|
||||
"grad_norm": 2.65625,
|
||||
"learning_rate": 5.78e-06,
|
||||
"loss": 0.741,
|
||||
"mean_token_accuracy": 0.8087894931435585,
|
||||
"num_tokens": 8271787.0,
|
||||
"step": 290
|
||||
},
|
||||
{
|
||||
"entropy": 0.7761816097423434,
|
||||
"epoch": 0.48,
|
||||
"grad_norm": 1.78125,
|
||||
"learning_rate": 5.98e-06,
|
||||
"loss": 0.7548,
|
||||
"mean_token_accuracy": 0.8063571337610483,
|
||||
"num_tokens": 8556170.0,
|
||||
"step": 300
|
||||
},
|
||||
{
|
||||
"epoch": 0.48,
|
||||
"eval_math_entropy": 0.7653279647827148,
|
||||
"eval_math_loss": 0.7636919021606445,
|
||||
"eval_math_mean_token_accuracy": 0.8056883907318115,
|
||||
"eval_math_num_tokens": 8556170.0,
|
||||
"eval_math_runtime": 35.8013,
|
||||
"eval_math_samples_per_second": 13.966,
|
||||
"eval_math_steps_per_second": 3.491,
|
||||
"step": 300
|
||||
},
|
||||
{
|
||||
"epoch": 0.48,
|
||||
"eval_physics_entropy": 0.6890090732574463,
|
||||
"eval_physics_loss": 0.6574941277503967,
|
||||
"eval_physics_mean_token_accuracy": 0.8227382025718689,
|
||||
"eval_physics_num_tokens": 8556170.0,
|
||||
"eval_physics_runtime": 41.4883,
|
||||
"eval_physics_samples_per_second": 12.052,
|
||||
"eval_physics_steps_per_second": 3.013,
|
||||
"step": 300
|
||||
},
|
||||
{
|
||||
"entropy": 0.7349074123427272,
|
||||
"epoch": 0.496,
|
||||
"grad_norm": 1.828125,
|
||||
"learning_rate": 6.18e-06,
|
||||
"loss": 0.7172,
|
||||
"mean_token_accuracy": 0.8132212206721305,
|
||||
"num_tokens": 8840590.0,
|
||||
"step": 310
|
||||
},
|
||||
{
|
||||
"entropy": 0.7443849379196763,
|
||||
"epoch": 0.512,
|
||||
"grad_norm": 1.6640625,
|
||||
"learning_rate": 6.380000000000001e-06,
|
||||
"loss": 0.7181,
|
||||
"mean_token_accuracy": 0.812743404135108,
|
||||
"num_tokens": 9131903.0,
|
||||
"step": 320
|
||||
},
|
||||
{
|
||||
"entropy": 0.7170431386679411,
|
||||
"epoch": 0.528,
|
||||
"grad_norm": 1.7109375,
|
||||
"learning_rate": 6.5800000000000005e-06,
|
||||
"loss": 0.7045,
|
||||
"mean_token_accuracy": 0.8160003747791051,
|
||||
"num_tokens": 9429519.0,
|
||||
"step": 330
|
||||
},
|
||||
{
|
||||
"entropy": 0.7328394843265414,
|
||||
"epoch": 0.544,
|
||||
"grad_norm": 2.234375,
|
||||
"learning_rate": 6.780000000000001e-06,
|
||||
"loss": 0.722,
|
||||
"mean_token_accuracy": 0.8143766384571791,
|
||||
"num_tokens": 9718247.0,
|
||||
"step": 340
|
||||
},
|
||||
{
|
||||
"entropy": 0.7620531113818287,
|
||||
"epoch": 0.56,
|
||||
"grad_norm": 2.453125,
|
||||
"learning_rate": 6.98e-06,
|
||||
"loss": 0.7395,
|
||||
"mean_token_accuracy": 0.8070962373167276,
|
||||
"num_tokens": 9991100.0,
|
||||
"step": 350
|
||||
},
|
||||
{
|
||||
"entropy": 0.7331368897110224,
|
||||
"epoch": 0.576,
|
||||
"grad_norm": 1.8515625,
|
||||
"learning_rate": 7.180000000000001e-06,
|
||||
"loss": 0.7173,
|
||||
"mean_token_accuracy": 0.8115411166101694,
|
||||
"num_tokens": 10276090.0,
|
||||
"step": 360
|
||||
},
|
||||
{
|
||||
"entropy": 0.7342592483386398,
|
||||
"epoch": 0.592,
|
||||
"grad_norm": 1.546875,
|
||||
"learning_rate": 7.3800000000000005e-06,
|
||||
"loss": 0.7141,
|
||||
"mean_token_accuracy": 0.8124774299561978,
|
||||
"num_tokens": 10566762.0,
|
||||
"step": 370
|
||||
},
|
||||
{
|
||||
"entropy": 0.7440765531733632,
|
||||
"epoch": 0.608,
|
||||
"grad_norm": 1.59375,
|
||||
"learning_rate": 7.58e-06,
|
||||
"loss": 0.7237,
|
||||
"mean_token_accuracy": 0.8122259072959424,
|
||||
"num_tokens": 10857002.0,
|
||||
"step": 380
|
||||
},
|
||||
{
|
||||
"entropy": 0.7216648861765862,
|
||||
"epoch": 0.624,
|
||||
"grad_norm": 2.25,
|
||||
"learning_rate": 7.78e-06,
|
||||
"loss": 0.7057,
|
||||
"mean_token_accuracy": 0.8139940846711398,
|
||||
"num_tokens": 11147326.0,
|
||||
"step": 390
|
||||
},
|
||||
{
|
||||
"entropy": 0.7119627617299557,
|
||||
"epoch": 0.64,
|
||||
"grad_norm": 1.5234375,
|
||||
"learning_rate": 7.980000000000002e-06,
|
||||
"loss": 0.6947,
|
||||
"mean_token_accuracy": 0.8168974690139293,
|
||||
"num_tokens": 11437710.0,
|
||||
"step": 400
|
||||
},
|
||||
{
|
||||
"epoch": 0.64,
|
||||
"eval_math_entropy": 0.7429183626174927,
|
||||
"eval_math_loss": 0.7330864071846008,
|
||||
"eval_math_mean_token_accuracy": 0.8096566972732544,
|
||||
"eval_math_num_tokens": 11437710.0,
|
||||
"eval_math_runtime": 35.7944,
|
||||
"eval_math_samples_per_second": 13.969,
|
||||
"eval_math_steps_per_second": 3.492,
|
||||
"step": 400
|
||||
},
|
||||
{
|
||||
"epoch": 0.64,
|
||||
"eval_physics_entropy": 0.6820782516002655,
|
||||
"eval_physics_loss": 0.6511842012405396,
|
||||
"eval_physics_mean_token_accuracy": 0.8221303143501282,
|
||||
"eval_physics_num_tokens": 11437710.0,
|
||||
"eval_physics_runtime": 41.4584,
|
||||
"eval_physics_samples_per_second": 12.06,
|
||||
"eval_physics_steps_per_second": 3.015,
|
||||
"step": 400
|
||||
},
|
||||
{
|
||||
"entropy": 0.712121813185513,
|
||||
"epoch": 0.656,
|
||||
"grad_norm": 1.40625,
|
||||
"learning_rate": 8.18e-06,
|
||||
"loss": 0.6914,
|
||||
"mean_token_accuracy": 0.8179911982268095,
|
||||
"num_tokens": 11727268.0,
|
||||
"step": 410
|
||||
},
|
||||
{
|
||||
"entropy": 0.7159143410623073,
|
||||
"epoch": 0.672,
|
||||
"grad_norm": 1.65625,
|
||||
"learning_rate": 8.380000000000001e-06,
|
||||
"loss": 0.7016,
|
||||
"mean_token_accuracy": 0.813546035438776,
|
||||
"num_tokens": 12020478.0,
|
||||
"step": 420
|
||||
},
|
||||
{
|
||||
"entropy": 0.7149255678057671,
|
||||
"epoch": 0.688,
|
||||
"grad_norm": 1.5078125,
|
||||
"learning_rate": 8.580000000000001e-06,
|
||||
"loss": 0.6908,
|
||||
"mean_token_accuracy": 0.8162215895950794,
|
||||
"num_tokens": 12300380.0,
|
||||
"step": 430
|
||||
},
|
||||
{
|
||||
"entropy": 0.7053035136312247,
|
||||
"epoch": 0.704,
|
||||
"grad_norm": 1.78125,
|
||||
"learning_rate": 8.78e-06,
|
||||
"loss": 0.6839,
|
||||
"mean_token_accuracy": 0.8172036681324244,
|
||||
"num_tokens": 12591366.0,
|
||||
"step": 440
|
||||
},
|
||||
{
|
||||
"entropy": 0.7228720126673579,
|
||||
"epoch": 0.72,
|
||||
"grad_norm": 1.7421875,
|
||||
"learning_rate": 8.98e-06,
|
||||
"loss": 0.6945,
|
||||
"mean_token_accuracy": 0.8145164046436548,
|
||||
"num_tokens": 12869618.0,
|
||||
"step": 450
|
||||
},
|
||||
{
|
||||
"entropy": 0.7052439017221331,
|
||||
"epoch": 0.736,
|
||||
"grad_norm": 1.75,
|
||||
"learning_rate": 9.180000000000002e-06,
|
||||
"loss": 0.6866,
|
||||
"mean_token_accuracy": 0.8176059562712907,
|
||||
"num_tokens": 13156311.0,
|
||||
"step": 460
|
||||
},
|
||||
{
|
||||
"entropy": 0.692553037032485,
|
||||
"epoch": 0.752,
|
||||
"grad_norm": 1.5625,
|
||||
"learning_rate": 9.38e-06,
|
||||
"loss": 0.6644,
|
||||
"mean_token_accuracy": 0.821098442748189,
|
||||
"num_tokens": 13438911.0,
|
||||
"step": 470
|
||||
},
|
||||
{
|
||||
"entropy": 0.6714967370033265,
|
||||
"epoch": 0.768,
|
||||
"grad_norm": 1.3203125,
|
||||
"learning_rate": 9.58e-06,
|
||||
"loss": 0.6595,
|
||||
"mean_token_accuracy": 0.825907539948821,
|
||||
"num_tokens": 13735397.0,
|
||||
"step": 480
|
||||
},
|
||||
{
|
||||
"entropy": 0.7148594724014401,
|
||||
"epoch": 0.784,
|
||||
"grad_norm": 1.7890625,
|
||||
"learning_rate": 9.780000000000001e-06,
|
||||
"loss": 0.6954,
|
||||
"mean_token_accuracy": 0.8156244982033968,
|
||||
"num_tokens": 14024497.0,
|
||||
"step": 490
|
||||
},
|
||||
{
|
||||
"entropy": 0.7110015576705336,
|
||||
"epoch": 0.8,
|
||||
"grad_norm": 1.6328125,
|
||||
"learning_rate": 9.980000000000001e-06,
|
||||
"loss": 0.69,
|
||||
"mean_token_accuracy": 0.8156023971736431,
|
||||
"num_tokens": 14316427.0,
|
||||
"step": 500
|
||||
},
|
||||
{
|
||||
"epoch": 0.8,
|
||||
"eval_math_entropy": 0.7275117125511169,
|
||||
"eval_math_loss": 0.7106672525405884,
|
||||
"eval_math_mean_token_accuracy": 0.8132652158737183,
|
||||
"eval_math_num_tokens": 14316427.0,
|
||||
"eval_math_runtime": 35.7797,
|
||||
"eval_math_samples_per_second": 13.974,
|
||||
"eval_math_steps_per_second": 3.494,
|
||||
"step": 500
|
||||
},
|
||||
{
|
||||
"epoch": 0.8,
|
||||
"eval_physics_entropy": 0.6752480027675629,
|
||||
"eval_physics_loss": 0.6474179029464722,
|
||||
"eval_physics_mean_token_accuracy": 0.8222041974067688,
|
||||
"eval_physics_num_tokens": 14316427.0,
|
||||
"eval_physics_runtime": 41.1592,
|
||||
"eval_physics_samples_per_second": 12.148,
|
||||
"eval_physics_steps_per_second": 3.037,
|
||||
"step": 500
|
||||
}
|
||||
],
|
||||
"logging_steps": 10,
|
||||
"max_steps": 10000,
|
||||
"num_input_tokens_seen": 0,
|
||||
"num_train_epochs": 16,
|
||||
"save_steps": 500,
|
||||
"stateful_callbacks": {
|
||||
"TrainerControl": {
|
||||
"args": {
|
||||
"should_epoch_stop": false,
|
||||
"should_evaluate": false,
|
||||
"should_log": false,
|
||||
"should_save": true,
|
||||
"should_training_stop": false
|
||||
},
|
||||
"attributes": {}
|
||||
}
|
||||
},
|
||||
"total_flos": 1.0863379264774472e+18,
|
||||
"train_batch_size": 4,
|
||||
"trial_name": null,
|
||||
"trial_params": null
|
||||
}
|
||||
3
checkpoint-500/training_args.bin
Normal file
3
checkpoint-500/training_args.bin
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:0612b24a8c5a6d7aa7710b98af44bc59fdb9c09f7a5c26c78b87b56c67358eb7
|
||||
size 6609
|
||||
Reference in New Issue
Block a user