初始化项目，由ModelHub XC社区提供模型
Model: roonbug/64a5wv2p Source: Original Platform
This commit is contained in:
ModelHub XC
9
checkpoint-500/chat_template.jinja
Normal file
9
checkpoint-500/chat_template.jinja
Normal file
@@ -0,0 +1,9 @@
|
||||
{{ bos_token }}{% for message in messages %}{% if message['role'] == 'system' %}{{ '<|system|>
|
||||
' + message['content'] + '
|
||||
' }}{% elif message['role'] == 'user' %}{{ '<|user|>
|
||||
' + message['content'] + '
|
||||
' }}{% elif message['role'] == 'assistant' %}{% if not loop.last %}{{ '<|assistant|>
|
||||
' + message['content'] + eos_token + '
|
||||
' }}{% else %}{{ '<|assistant|>
|
||||
' + message['content'] + eos_token }}{% endif %}{% endif %}{% if loop.last and add_generation_prompt %}{{ '<|assistant|>
|
||||
' }}{% endif %}{% endfor %}
|
||||
27
checkpoint-500/config.json
Normal file
27
checkpoint-500/config.json
Normal file
@@ -0,0 +1,27 @@
|
||||
{
|
||||
"architectures": [
|
||||
"Olmo2ForCausalLM"
|
||||
],
|
||||
"attention_bias": false,
|
||||
"attention_dropout": 0.0,
|
||||
"bos_token_id": 100257,
|
||||
"dtype": "bfloat16",
|
||||
"eos_token_id": 100257,
|
||||
"hidden_act": "silu",
|
||||
"hidden_size": 4096,
|
||||
"initializer_range": 0.02,
|
||||
"intermediate_size": 11008,
|
||||
"max_position_embeddings": 4096,
|
||||
"model_type": "olmo2",
|
||||
"num_attention_heads": 32,
|
||||
"num_hidden_layers": 32,
|
||||
"num_key_value_heads": 32,
|
||||
"pad_token_id": 100277,
|
||||
"rms_norm_eps": 1e-06,
|
||||
"rope_scaling": null,
|
||||
"rope_theta": 500000,
|
||||
"tie_word_embeddings": false,
|
||||
"transformers_version": "4.56.1",
|
||||
"use_cache": false,
|
||||
"vocab_size": 100352
|
||||
}
|
||||
9
checkpoint-500/generation_config.json
Normal file
9
checkpoint-500/generation_config.json
Normal file
@@ -0,0 +1,9 @@
|
||||
{
|
||||
"_from_model_config": true,
|
||||
"bos_token_id": 100257,
|
||||
"eos_token_id": [
|
||||
100257
|
||||
],
|
||||
"pad_token_id": 100277,
|
||||
"transformers_version": "4.56.1"
|
||||
}
|
||||
100001
checkpoint-500/merges.txt
Normal file
100001
checkpoint-500/merges.txt
Normal file
File diff suppressed because it is too large
Load Diff
3
checkpoint-500/model-00001-of-00003.safetensors
Normal file
3
checkpoint-500/model-00001-of-00003.safetensors
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:05ae91e1171f1f1753b23df3a5a6740d89da377932b33a1661bb7485d6ec530e
|
||||
size 4970591184
|
||||
3
checkpoint-500/model-00002-of-00003.safetensors
Normal file
3
checkpoint-500/model-00002-of-00003.safetensors
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:0d41b81fe0d7e4c1d673a77beded5ee77080e39d6a6940b523e82f61eaa1dcbd
|
||||
size 4981161496
|
||||
3
checkpoint-500/model-00003-of-00003.safetensors
Normal file
3
checkpoint-500/model-00003-of-00003.safetensors
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:40836f0ed0e71c57b12d8588ed1578838c1a329fd15da6b690f5064babc8e162
|
||||
size 4645523448
|
||||
363
checkpoint-500/model.safetensors.index.json
Normal file
363
checkpoint-500/model.safetensors.index.json
Normal file
@@ -0,0 +1,363 @@
|
||||
{
|
||||
"metadata": {
|
||||
"total_parameters": 7298617344,
|
||||
"total_size": 14597234688
|
||||
},
|
||||
"weight_map": {
|
||||
"lm_head.weight": "model-00003-of-00003.safetensors",
|
||||
"model.embed_tokens.weight": "model-00001-of-00003.safetensors",
|
||||
"model.layers.0.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
|
||||
"model.layers.0.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
|
||||
"model.layers.0.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
|
||||
"model.layers.0.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
|
||||
"model.layers.0.post_feedforward_layernorm.weight": "model-00001-of-00003.safetensors",
|
||||
"model.layers.0.self_attn.k_norm.weight": "model-00001-of-00003.safetensors",
|
||||
"model.layers.0.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
|
||||
"model.layers.0.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
|
||||
"model.layers.0.self_attn.q_norm.weight": "model-00001-of-00003.safetensors",
|
||||
"model.layers.0.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
|
||||
"model.layers.0.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
|
||||
"model.layers.1.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
|
||||
"model.layers.1.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
|
||||
"model.layers.1.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
|
||||
"model.layers.1.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
|
||||
"model.layers.1.post_feedforward_layernorm.weight": "model-00001-of-00003.safetensors",
|
||||
"model.layers.1.self_attn.k_norm.weight": "model-00001-of-00003.safetensors",
|
||||
"model.layers.1.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
|
||||
"model.layers.1.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
|
||||
"model.layers.1.self_attn.q_norm.weight": "model-00001-of-00003.safetensors",
|
||||
"model.layers.1.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
|
||||
"model.layers.1.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
|
||||
"model.layers.10.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
|
||||
"model.layers.10.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
|
||||
"model.layers.10.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
|
||||
"model.layers.10.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
|
||||
"model.layers.10.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors",
|
||||
"model.layers.10.self_attn.k_norm.weight": "model-00002-of-00003.safetensors",
|
||||
"model.layers.10.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
|
||||
"model.layers.10.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
|
||||
"model.layers.10.self_attn.q_norm.weight": "model-00002-of-00003.safetensors",
|
||||
"model.layers.10.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
|
||||
"model.layers.10.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
|
||||
"model.layers.11.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
|
||||
"model.layers.11.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
|
||||
"model.layers.11.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
|
||||
"model.layers.11.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
|
||||
"model.layers.11.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors",
|
||||
"model.layers.11.self_attn.k_norm.weight": "model-00002-of-00003.safetensors",
|
||||
"model.layers.11.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
|
||||
"model.layers.11.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
|
||||
"model.layers.11.self_attn.q_norm.weight": "model-00002-of-00003.safetensors",
|
||||
"model.layers.11.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
|
||||
"model.layers.11.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
|
||||
"model.layers.12.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
|
||||
"model.layers.12.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
|
||||
"model.layers.12.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
|
||||
"model.layers.12.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
|
||||
"model.layers.12.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors",
|
||||
"model.layers.12.self_attn.k_norm.weight": "model-00002-of-00003.safetensors",
|
||||
"model.layers.12.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
|
||||
"model.layers.12.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
|
||||
"model.layers.12.self_attn.q_norm.weight": "model-00002-of-00003.safetensors",
|
||||
"model.layers.12.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
|
||||
"model.layers.12.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
|
||||
"model.layers.13.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
|
||||
"model.layers.13.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
|
||||
"model.layers.13.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
|
||||
"model.layers.13.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
|
||||
"model.layers.13.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors",
|
||||
"model.layers.13.self_attn.k_norm.weight": "model-00002-of-00003.safetensors",
|
||||
"model.layers.13.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
|
||||
"model.layers.13.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
|
||||
"model.layers.13.self_attn.q_norm.weight": "model-00002-of-00003.safetensors",
|
||||
"model.layers.13.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
|
||||
"model.layers.13.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
|
||||
"model.layers.14.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
|
||||
"model.layers.14.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
|
||||
"model.layers.14.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
|
||||
"model.layers.14.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
|
||||
"model.layers.14.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors",
|
||||
"model.layers.14.self_attn.k_norm.weight": "model-00002-of-00003.safetensors",
|
||||
"model.layers.14.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
|
||||
"model.layers.14.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
|
||||
"model.layers.14.self_attn.q_norm.weight": "model-00002-of-00003.safetensors",
|
||||
"model.layers.14.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
|
||||
"model.layers.14.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
|
||||
"model.layers.15.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
|
||||
"model.layers.15.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
|
||||
"model.layers.15.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
|
||||
"model.layers.15.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
|
||||
"model.layers.15.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors",
|
||||
"model.layers.15.self_attn.k_norm.weight": "model-00002-of-00003.safetensors",
|
||||
"model.layers.15.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
|
||||
"model.layers.15.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
|
||||
"model.layers.15.self_attn.q_norm.weight": "model-00002-of-00003.safetensors",
|
||||
"model.layers.15.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
|
||||
"model.layers.15.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
|
||||
"model.layers.16.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
|
||||
"model.layers.16.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
|
||||
"model.layers.16.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
|
||||
"model.layers.16.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
|
||||
"model.layers.16.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors",
|
||||
"model.layers.16.self_attn.k_norm.weight": "model-00002-of-00003.safetensors",
|
||||
"model.layers.16.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
|
||||
"model.layers.16.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
|
||||
"model.layers.16.self_attn.q_norm.weight": "model-00002-of-00003.safetensors",
|
||||
"model.layers.16.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
|
||||
"model.layers.16.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
|
||||
"model.layers.17.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
|
||||
"model.layers.17.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
|
||||
"model.layers.17.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
|
||||
"model.layers.17.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
|
||||
"model.layers.17.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors",
|
||||
"model.layers.17.self_attn.k_norm.weight": "model-00002-of-00003.safetensors",
|
||||
"model.layers.17.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
|
||||
"model.layers.17.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
|
||||
"model.layers.17.self_attn.q_norm.weight": "model-00002-of-00003.safetensors",
|
||||
"model.layers.17.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
|
||||
"model.layers.17.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
|
||||
"model.layers.18.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
|
||||
"model.layers.18.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
|
||||
"model.layers.18.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
|
||||
"model.layers.18.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
|
||||
"model.layers.18.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors",
|
||||
"model.layers.18.self_attn.k_norm.weight": "model-00002-of-00003.safetensors",
|
||||
"model.layers.18.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
|
||||
"model.layers.18.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
|
||||
"model.layers.18.self_attn.q_norm.weight": "model-00002-of-00003.safetensors",
|
||||
"model.layers.18.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
|
||||
"model.layers.18.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
|
||||
"model.layers.19.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
|
||||
"model.layers.19.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
|
||||
"model.layers.19.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
|
||||
"model.layers.19.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
|
||||
"model.layers.19.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors",
|
||||
"model.layers.19.self_attn.k_norm.weight": "model-00002-of-00003.safetensors",
|
||||
"model.layers.19.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
|
||||
"model.layers.19.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
|
||||
"model.layers.19.self_attn.q_norm.weight": "model-00002-of-00003.safetensors",
|
||||
"model.layers.19.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
|
||||
"model.layers.19.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
|
||||
"model.layers.2.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
|
||||
"model.layers.2.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
|
||||
"model.layers.2.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
|
||||
"model.layers.2.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
|
||||
"model.layers.2.post_feedforward_layernorm.weight": "model-00001-of-00003.safetensors",
|
||||
"model.layers.2.self_attn.k_norm.weight": "model-00001-of-00003.safetensors",
|
||||
"model.layers.2.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
|
||||
"model.layers.2.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
|
||||
"model.layers.2.self_attn.q_norm.weight": "model-00001-of-00003.safetensors",
|
||||
"model.layers.2.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
|
||||
"model.layers.2.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
|
||||
"model.layers.20.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
|
||||
"model.layers.20.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
|
||||
"model.layers.20.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
|
||||
"model.layers.20.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
|
||||
"model.layers.20.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors",
|
||||
"model.layers.20.self_attn.k_norm.weight": "model-00002-of-00003.safetensors",
|
||||
"model.layers.20.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
|
||||
"model.layers.20.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
|
||||
"model.layers.20.self_attn.q_norm.weight": "model-00002-of-00003.safetensors",
|
||||
"model.layers.20.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
|
||||
"model.layers.20.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
|
||||
"model.layers.21.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
|
||||
"model.layers.21.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
|
||||
"model.layers.21.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
|
||||
"model.layers.21.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
|
||||
"model.layers.21.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors",
|
||||
"model.layers.21.self_attn.k_norm.weight": "model-00002-of-00003.safetensors",
|
||||
"model.layers.21.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
|
||||
"model.layers.21.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
|
||||
"model.layers.21.self_attn.q_norm.weight": "model-00002-of-00003.safetensors",
|
||||
"model.layers.21.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
|
||||
"model.layers.21.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
|
||||
"model.layers.22.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
|
||||
"model.layers.22.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
|
||||
"model.layers.22.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
|
||||
"model.layers.22.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
|
||||
"model.layers.22.post_feedforward_layernorm.weight": "model-00003-of-00003.safetensors",
|
||||
"model.layers.22.self_attn.k_norm.weight": "model-00002-of-00003.safetensors",
|
||||
"model.layers.22.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
|
||||
"model.layers.22.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
|
||||
"model.layers.22.self_attn.q_norm.weight": "model-00002-of-00003.safetensors",
|
||||
"model.layers.22.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
|
||||
"model.layers.22.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
|
||||
"model.layers.23.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
|
||||
"model.layers.23.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
|
||||
"model.layers.23.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
|
||||
"model.layers.23.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
|
||||
"model.layers.23.post_feedforward_layernorm.weight": "model-00003-of-00003.safetensors",
|
||||
"model.layers.23.self_attn.k_norm.weight": "model-00003-of-00003.safetensors",
|
||||
"model.layers.23.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
|
||||
"model.layers.23.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
|
||||
"model.layers.23.self_attn.q_norm.weight": "model-00003-of-00003.safetensors",
|
||||
"model.layers.23.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
|
||||
"model.layers.23.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
|
||||
"model.layers.24.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
|
||||
"model.layers.24.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
|
||||
"model.layers.24.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
|
||||
"model.layers.24.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
|
||||
"model.layers.24.post_feedforward_layernorm.weight": "model-00003-of-00003.safetensors",
|
||||
"model.layers.24.self_attn.k_norm.weight": "model-00003-of-00003.safetensors",
|
||||
"model.layers.24.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
|
||||
"model.layers.24.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
|
||||
"model.layers.24.self_attn.q_norm.weight": "model-00003-of-00003.safetensors",
|
||||
"model.layers.24.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
|
||||
"model.layers.24.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
|
||||
"model.layers.25.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
|
||||
"model.layers.25.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
|
||||
"model.layers.25.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
|
||||
"model.layers.25.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
|
||||
"model.layers.25.post_feedforward_layernorm.weight": "model-00003-of-00003.safetensors",
|
||||
"model.layers.25.self_attn.k_norm.weight": "model-00003-of-00003.safetensors",
|
||||
"model.layers.25.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
|
||||
"model.layers.25.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
|
||||
"model.layers.25.self_attn.q_norm.weight": "model-00003-of-00003.safetensors",
|
||||
"model.layers.25.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
|
||||
"model.layers.25.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
|
||||
"model.layers.26.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
|
||||
"model.layers.26.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
|
||||
"model.layers.26.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
|
||||
"model.layers.26.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
|
||||
"model.layers.26.post_feedforward_layernorm.weight": "model-00003-of-00003.safetensors",
|
||||
"model.layers.26.self_attn.k_norm.weight": "model-00003-of-00003.safetensors",
|
||||
"model.layers.26.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
|
||||
"model.layers.26.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
|
||||
"model.layers.26.self_attn.q_norm.weight": "model-00003-of-00003.safetensors",
|
||||
"model.layers.26.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
|
||||
"model.layers.26.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
|
||||
"model.layers.27.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
|
||||
"model.layers.27.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
|
||||
"model.layers.27.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
|
||||
"model.layers.27.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
|
||||
"model.layers.27.post_feedforward_layernorm.weight": "model-00003-of-00003.safetensors",
|
||||
"model.layers.27.self_attn.k_norm.weight": "model-00003-of-00003.safetensors",
|
||||
"model.layers.27.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
|
||||
"model.layers.27.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
|
||||
"model.layers.27.self_attn.q_norm.weight": "model-00003-of-00003.safetensors",
|
||||
"model.layers.27.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
|
||||
"model.layers.27.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
|
||||
"model.layers.28.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
|
||||
"model.layers.28.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
|
||||
"model.layers.28.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
|
||||
"model.layers.28.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
|
||||
"model.layers.28.post_feedforward_layernorm.weight": "model-00003-of-00003.safetensors",
|
||||
"model.layers.28.self_attn.k_norm.weight": "model-00003-of-00003.safetensors",
|
||||
"model.layers.28.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
|
||||
"model.layers.28.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
|
||||
"model.layers.28.self_attn.q_norm.weight": "model-00003-of-00003.safetensors",
|
||||
"model.layers.28.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
|
||||
"model.layers.28.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
|
||||
"model.layers.29.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
|
||||
"model.layers.29.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
|
||||
"model.layers.29.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
|
||||
"model.layers.29.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
|
||||
"model.layers.29.post_feedforward_layernorm.weight": "model-00003-of-00003.safetensors",
|
||||
"model.layers.29.self_attn.k_norm.weight": "model-00003-of-00003.safetensors",
|
||||
"model.layers.29.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
|
||||
"model.layers.29.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
|
||||
"model.layers.29.self_attn.q_norm.weight": "model-00003-of-00003.safetensors",
|
||||
"model.layers.29.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
|
||||
"model.layers.29.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
|
||||
"model.layers.3.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
|
||||
"model.layers.3.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
|
||||
"model.layers.3.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
|
||||
"model.layers.3.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
|
||||
"model.layers.3.post_feedforward_layernorm.weight": "model-00001-of-00003.safetensors",
|
||||
"model.layers.3.self_attn.k_norm.weight": "model-00001-of-00003.safetensors",
|
||||
"model.layers.3.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
|
||||
"model.layers.3.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
|
||||
"model.layers.3.self_attn.q_norm.weight": "model-00001-of-00003.safetensors",
|
||||
"model.layers.3.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
|
||||
"model.layers.3.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
|
||||
"model.layers.30.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
|
||||
"model.layers.30.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
|
||||
"model.layers.30.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
|
||||
"model.layers.30.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
|
||||
"model.layers.30.post_feedforward_layernorm.weight": "model-00003-of-00003.safetensors",
|
||||
"model.layers.30.self_attn.k_norm.weight": "model-00003-of-00003.safetensors",
|
||||
"model.layers.30.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
|
||||
"model.layers.30.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
|
||||
"model.layers.30.self_attn.q_norm.weight": "model-00003-of-00003.safetensors",
|
||||
"model.layers.30.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
|
||||
"model.layers.30.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
|
||||
"model.layers.31.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
|
||||
"model.layers.31.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
|
||||
"model.layers.31.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
|
||||
"model.layers.31.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
|
||||
"model.layers.31.post_feedforward_layernorm.weight": "model-00003-of-00003.safetensors",
|
||||
"model.layers.31.self_attn.k_norm.weight": "model-00003-of-00003.safetensors",
|
||||
"model.layers.31.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
|
||||
"model.layers.31.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
|
||||
"model.layers.31.self_attn.q_norm.weight": "model-00003-of-00003.safetensors",
|
||||
"model.layers.31.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
|
||||
"model.layers.31.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
|
||||
"model.layers.4.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
|
||||
"model.layers.4.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
|
||||
"model.layers.4.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
|
||||
"model.layers.4.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
|
||||
"model.layers.4.post_feedforward_layernorm.weight": "model-00001-of-00003.safetensors",
|
||||
"model.layers.4.self_attn.k_norm.weight": "model-00001-of-00003.safetensors",
|
||||
"model.layers.4.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
|
||||
"model.layers.4.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
|
||||
"model.layers.4.self_attn.q_norm.weight": "model-00001-of-00003.safetensors",
|
||||
"model.layers.4.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
|
||||
"model.layers.4.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
|
||||
"model.layers.5.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
|
||||
"model.layers.5.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
|
||||
"model.layers.5.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
|
||||
"model.layers.5.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
|
||||
"model.layers.5.post_feedforward_layernorm.weight": "model-00001-of-00003.safetensors",
|
||||
"model.layers.5.self_attn.k_norm.weight": "model-00001-of-00003.safetensors",
|
||||
"model.layers.5.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
|
||||
"model.layers.5.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
|
||||
"model.layers.5.self_attn.q_norm.weight": "model-00001-of-00003.safetensors",
|
||||
"model.layers.5.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
|
||||
"model.layers.5.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
|
||||
"model.layers.6.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
|
||||
"model.layers.6.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
|
||||
"model.layers.6.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
|
||||
"model.layers.6.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
|
||||
"model.layers.6.post_feedforward_layernorm.weight": "model-00001-of-00003.safetensors",
|
||||
"model.layers.6.self_attn.k_norm.weight": "model-00001-of-00003.safetensors",
|
||||
"model.layers.6.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
|
||||
"model.layers.6.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
|
||||
"model.layers.6.self_attn.q_norm.weight": "model-00001-of-00003.safetensors",
|
||||
"model.layers.6.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
|
||||
"model.layers.6.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
|
||||
"model.layers.7.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
|
||||
"model.layers.7.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
|
||||
"model.layers.7.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
|
||||
"model.layers.7.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
|
||||
"model.layers.7.post_feedforward_layernorm.weight": "model-00001-of-00003.safetensors",
|
||||
"model.layers.7.self_attn.k_norm.weight": "model-00001-of-00003.safetensors",
|
||||
"model.layers.7.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
|
||||
"model.layers.7.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
|
||||
"model.layers.7.self_attn.q_norm.weight": "model-00001-of-00003.safetensors",
|
||||
"model.layers.7.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
|
||||
"model.layers.7.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
|
||||
"model.layers.8.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
|
||||
"model.layers.8.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
|
||||
"model.layers.8.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
|
||||
"model.layers.8.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
|
||||
"model.layers.8.post_feedforward_layernorm.weight": "model-00001-of-00003.safetensors",
|
||||
"model.layers.8.self_attn.k_norm.weight": "model-00001-of-00003.safetensors",
|
||||
"model.layers.8.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
|
||||
"model.layers.8.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
|
||||
"model.layers.8.self_attn.q_norm.weight": "model-00001-of-00003.safetensors",
|
||||
"model.layers.8.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
|
||||
"model.layers.8.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
|
||||
"model.layers.9.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
|
||||
"model.layers.9.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
|
||||
"model.layers.9.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
|
||||
"model.layers.9.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
|
||||
"model.layers.9.post_feedforward_layernorm.weight": "model-00001-of-00003.safetensors",
|
||||
"model.layers.9.self_attn.k_norm.weight": "model-00001-of-00003.safetensors",
|
||||
"model.layers.9.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
|
||||
"model.layers.9.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
|
||||
"model.layers.9.self_attn.q_norm.weight": "model-00001-of-00003.safetensors",
|
||||
"model.layers.9.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
|
||||
"model.layers.9.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
|
||||
"model.norm.weight": "model-00003-of-00003.safetensors"
|
||||
}
|
||||
}
|
||||
3
checkpoint-500/optimizer.pt
Normal file
3
checkpoint-500/optimizer.pt
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:9fe6bb103db9dce4a3edbec7b1b5df8cfd7c2116580fc500e5bf4677d723c581
|
||||
size 3263319914
|
||||
3
checkpoint-500/rng_state.pth
Normal file
3
checkpoint-500/rng_state.pth
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:fde986f97ecc270a34049538e5bcb495edbc4e7f5914a76b3cad083de287dc73
|
||||
size 14645
|
||||
3
checkpoint-500/scheduler.pt
Normal file
3
checkpoint-500/scheduler.pt
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:ddc00081d4f892d131a0d2b4d4827a214275362bfa62dfb8413f489c1bfad457
|
||||
size 1465
|
||||
30
checkpoint-500/special_tokens_map.json
Normal file
30
checkpoint-500/special_tokens_map.json
Normal file
@@ -0,0 +1,30 @@
|
||||
{
|
||||
"bos_token": {
|
||||
"content": "<|endoftext|>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false
|
||||
},
|
||||
"eos_token": {
|
||||
"content": "<|endoftext|>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false
|
||||
},
|
||||
"pad_token": {
|
||||
"content": "<|pad|>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false
|
||||
},
|
||||
"unk_token": {
|
||||
"content": "<|endoftext|>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false
|
||||
}
|
||||
}
|
||||
500524
checkpoint-500/tokenizer.json
Normal file
500524
checkpoint-500/tokenizer.json
Normal file
File diff suppressed because it is too large
Load Diff
189
checkpoint-500/tokenizer_config.json
Normal file
189
checkpoint-500/tokenizer_config.json
Normal file
@@ -0,0 +1,189 @@
|
||||
{
|
||||
"add_prefix_space": false,
|
||||
"added_tokens_decoder": {
|
||||
"100256": {
|
||||
"content": "<|extra_id_0|>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": false
|
||||
},
|
||||
"100257": {
|
||||
"content": "<|endoftext|>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"100258": {
|
||||
"content": "<|fim_prefix|>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"100259": {
|
||||
"content": "<|fim_middle|>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"100260": {
|
||||
"content": "<|fim_suffix|>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"100261": {
|
||||
"content": "|||PHONE_NUMBER|||",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": false
|
||||
},
|
||||
"100262": {
|
||||
"content": "|||EMAIL_ADDRESS|||",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": false
|
||||
},
|
||||
"100263": {
|
||||
"content": "|||IP_ADDRESS|||",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": false
|
||||
},
|
||||
"100264": {
|
||||
"content": "<|im_start|>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"100265": {
|
||||
"content": "<|im_end|>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"100266": {
|
||||
"content": "<|extra_id_1|>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": false
|
||||
},
|
||||
"100267": {
|
||||
"content": "<|extra_id_2|>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": false
|
||||
},
|
||||
"100268": {
|
||||
"content": "<|extra_id_3|>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": false
|
||||
},
|
||||
"100269": {
|
||||
"content": "<|extra_id_4|>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": false
|
||||
},
|
||||
"100270": {
|
||||
"content": "<|extra_id_5|>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": false
|
||||
},
|
||||
"100271": {
|
||||
"content": "<|extra_id_6|>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": false
|
||||
},
|
||||
"100272": {
|
||||
"content": "<|extra_id_7|>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": false
|
||||
},
|
||||
"100273": {
|
||||
"content": "<|extra_id_8|>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": false
|
||||
},
|
||||
"100274": {
|
||||
"content": "<|extra_id_9|>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": false
|
||||
},
|
||||
"100275": {
|
||||
"content": "<|extra_id_10|>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": false
|
||||
},
|
||||
"100276": {
|
||||
"content": "<|endofprompt|>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"100277": {
|
||||
"content": "<|pad|>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
}
|
||||
},
|
||||
"bos_token": "<|endoftext|>",
|
||||
"clean_up_tokenization_spaces": false,
|
||||
"eos_token": "<|endoftext|>",
|
||||
"extra_special_tokens": {},
|
||||
"model_max_length": 1000000000000000019884624838656,
|
||||
"pad_token": "<|pad|>",
|
||||
"tokenizer_class": "GPT2Tokenizer",
|
||||
"unk_token": "<|endoftext|>"
|
||||
}
|
||||
754
checkpoint-500/trainer_state.json
Normal file
754
checkpoint-500/trainer_state.json
Normal file
@@ -0,0 +1,754 @@
|
||||
{
|
||||
"best_global_step": null,
|
||||
"best_metric": null,
|
||||
"best_model_checkpoint": null,
|
||||
"epoch": 0.4,
|
||||
"eval_steps": 100,
|
||||
"global_step": 500,
|
||||
"is_hyper_param_search": false,
|
||||
"is_local_process_zero": true,
|
||||
"is_world_process_zero": true,
|
||||
"log_history": [
|
||||
{
|
||||
"entropy": 0.6430254079401493,
|
||||
"epoch": 0.008,
|
||||
"grad_norm": 4.53125,
|
||||
"learning_rate": 6.000000000000001e-07,
|
||||
"loss": 1.8435,
|
||||
"mean_token_accuracy": 0.6683766476809978,
|
||||
"num_tokens": 133939.0,
|
||||
"step": 10
|
||||
},
|
||||
{
|
||||
"entropy": 0.6358832061290741,
|
||||
"epoch": 0.016,
|
||||
"grad_norm": 4.3125,
|
||||
"learning_rate": 1.2666666666666669e-06,
|
||||
"loss": 1.8392,
|
||||
"mean_token_accuracy": 0.6651193369179964,
|
||||
"num_tokens": 273600.0,
|
||||
"step": 20
|
||||
},
|
||||
{
|
||||
"entropy": 0.6754261270165444,
|
||||
"epoch": 0.024,
|
||||
"grad_norm": 4.9375,
|
||||
"learning_rate": 1.9333333333333336e-06,
|
||||
"loss": 1.9101,
|
||||
"mean_token_accuracy": 0.6521823685616255,
|
||||
"num_tokens": 408034.0,
|
||||
"step": 30
|
||||
},
|
||||
{
|
||||
"entropy": 0.6558344909921289,
|
||||
"epoch": 0.032,
|
||||
"grad_norm": 4.28125,
|
||||
"learning_rate": 2.6e-06,
|
||||
"loss": 1.8472,
|
||||
"mean_token_accuracy": 0.6629668604582548,
|
||||
"num_tokens": 542836.0,
|
||||
"step": 40
|
||||
},
|
||||
{
|
||||
"entropy": 0.6494104014709592,
|
||||
"epoch": 0.04,
|
||||
"grad_norm": 3.8125,
|
||||
"learning_rate": 3.266666666666667e-06,
|
||||
"loss": 1.8027,
|
||||
"mean_token_accuracy": 0.6676773574203253,
|
||||
"num_tokens": 678419.0,
|
||||
"step": 50
|
||||
},
|
||||
{
|
||||
"entropy": 0.662838158942759,
|
||||
"epoch": 0.048,
|
||||
"grad_norm": 4.03125,
|
||||
"learning_rate": 3.9333333333333335e-06,
|
||||
"loss": 1.8213,
|
||||
"mean_token_accuracy": 0.6647731238976121,
|
||||
"num_tokens": 813510.0,
|
||||
"step": 60
|
||||
},
|
||||
{
|
||||
"entropy": 0.6506876813247799,
|
||||
"epoch": 0.056,
|
||||
"grad_norm": 3.71875,
|
||||
"learning_rate": 4.600000000000001e-06,
|
||||
"loss": 1.7955,
|
||||
"mean_token_accuracy": 0.6665123915299773,
|
||||
"num_tokens": 954682.0,
|
||||
"step": 70
|
||||
},
|
||||
{
|
||||
"entropy": 0.6252049600705505,
|
||||
"epoch": 0.064,
|
||||
"grad_norm": 3.65625,
|
||||
"learning_rate": 5.2666666666666665e-06,
|
||||
"loss": 1.7861,
|
||||
"mean_token_accuracy": 0.6728611093014478,
|
||||
"num_tokens": 1096071.0,
|
||||
"step": 80
|
||||
},
|
||||
{
|
||||
"entropy": 0.6236795110628008,
|
||||
"epoch": 0.072,
|
||||
"grad_norm": 3.171875,
|
||||
"learning_rate": 5.933333333333335e-06,
|
||||
"loss": 1.6458,
|
||||
"mean_token_accuracy": 0.6831523377448321,
|
||||
"num_tokens": 1239910.0,
|
||||
"step": 90
|
||||
},
|
||||
{
|
||||
"entropy": 0.6565364092588425,
|
||||
"epoch": 0.08,
|
||||
"grad_norm": 3.125,
|
||||
"learning_rate": 6.600000000000001e-06,
|
||||
"loss": 1.6948,
|
||||
"mean_token_accuracy": 0.6728412076830864,
|
||||
"num_tokens": 1376045.0,
|
||||
"step": 100
|
||||
},
|
||||
{
|
||||
"epoch": 0.08,
|
||||
"eval_biology_entropy": 1.821127786874771,
|
||||
"eval_biology_loss": 4.8697123527526855,
|
||||
"eval_biology_mean_token_accuracy": 0.3407035154700279,
|
||||
"eval_biology_num_tokens": 1376045.0,
|
||||
"eval_biology_runtime": 26.9612,
|
||||
"eval_biology_samples_per_second": 18.545,
|
||||
"eval_biology_steps_per_second": 9.273,
|
||||
"step": 100
|
||||
},
|
||||
{
|
||||
"epoch": 0.08,
|
||||
"eval_chemistry_entropy": 1.1214371861219405,
|
||||
"eval_chemistry_loss": 2.9583740234375,
|
||||
"eval_chemistry_mean_token_accuracy": 0.5230503298044205,
|
||||
"eval_chemistry_num_tokens": 1376045.0,
|
||||
"eval_chemistry_runtime": 34.4743,
|
||||
"eval_chemistry_samples_per_second": 14.504,
|
||||
"eval_chemistry_steps_per_second": 7.252,
|
||||
"step": 100
|
||||
},
|
||||
{
|
||||
"epoch": 0.08,
|
||||
"eval_math_entropy": 0.6455623766183853,
|
||||
"eval_math_loss": 1.6610009670257568,
|
||||
"eval_math_mean_token_accuracy": 0.6816041300296783,
|
||||
"eval_math_num_tokens": 1376045.0,
|
||||
"eval_math_runtime": 34.9863,
|
||||
"eval_math_samples_per_second": 14.291,
|
||||
"eval_math_steps_per_second": 7.146,
|
||||
"step": 100
|
||||
},
|
||||
{
|
||||
"epoch": 0.08,
|
||||
"eval_physics_entropy": 0.8894976880550385,
|
||||
"eval_physics_loss": 2.340852975845337,
|
||||
"eval_physics_mean_token_accuracy": 0.5970065190792083,
|
||||
"eval_physics_num_tokens": 1376045.0,
|
||||
"eval_physics_runtime": 40.5871,
|
||||
"eval_physics_samples_per_second": 12.319,
|
||||
"eval_physics_steps_per_second": 6.16,
|
||||
"step": 100
|
||||
},
|
||||
{
|
||||
"entropy": 0.6108825646340847,
|
||||
"epoch": 0.088,
|
||||
"grad_norm": 3.0625,
|
||||
"learning_rate": 7.266666666666668e-06,
|
||||
"loss": 1.534,
|
||||
"mean_token_accuracy": 0.6979989364743233,
|
||||
"num_tokens": 1518257.0,
|
||||
"step": 110
|
||||
},
|
||||
{
|
||||
"entropy": 0.6626681972295045,
|
||||
"epoch": 0.096,
|
||||
"grad_norm": 3.0,
|
||||
"learning_rate": 7.933333333333334e-06,
|
||||
"loss": 1.4858,
|
||||
"mean_token_accuracy": 0.6921830836683511,
|
||||
"num_tokens": 1656215.0,
|
||||
"step": 120
|
||||
},
|
||||
{
|
||||
"entropy": 0.6807309044525027,
|
||||
"epoch": 0.104,
|
||||
"grad_norm": 2.4375,
|
||||
"learning_rate": 8.6e-06,
|
||||
"loss": 1.3897,
|
||||
"mean_token_accuracy": 0.7023331839591265,
|
||||
"num_tokens": 1791507.0,
|
||||
"step": 130
|
||||
},
|
||||
{
|
||||
"entropy": 0.7359636599197984,
|
||||
"epoch": 0.112,
|
||||
"grad_norm": 1.9765625,
|
||||
"learning_rate": 9.266666666666667e-06,
|
||||
"loss": 1.3356,
|
||||
"mean_token_accuracy": 0.7036307867616415,
|
||||
"num_tokens": 1928153.0,
|
||||
"step": 140
|
||||
},
|
||||
{
|
||||
"entropy": 0.8070579996332526,
|
||||
"epoch": 0.12,
|
||||
"grad_norm": 1.7265625,
|
||||
"learning_rate": 9.933333333333334e-06,
|
||||
"loss": 1.2682,
|
||||
"mean_token_accuracy": 0.712115453183651,
|
||||
"num_tokens": 2071008.0,
|
||||
"step": 150
|
||||
},
|
||||
{
|
||||
"entropy": 0.874446533434093,
|
||||
"epoch": 0.128,
|
||||
"grad_norm": 1.2734375,
|
||||
"learning_rate": 1.0600000000000002e-05,
|
||||
"loss": 1.1443,
|
||||
"mean_token_accuracy": 0.7246435590088367,
|
||||
"num_tokens": 2216042.0,
|
||||
"step": 160
|
||||
},
|
||||
{
|
||||
"entropy": 0.9639016635715961,
|
||||
"epoch": 0.136,
|
||||
"grad_norm": 1.125,
|
||||
"learning_rate": 1.1266666666666668e-05,
|
||||
"loss": 1.1005,
|
||||
"mean_token_accuracy": 0.730505557358265,
|
||||
"num_tokens": 2350589.0,
|
||||
"step": 170
|
||||
},
|
||||
{
|
||||
"entropy": 1.0533128878101707,
|
||||
"epoch": 0.144,
|
||||
"grad_norm": 0.90234375,
|
||||
"learning_rate": 1.1933333333333335e-05,
|
||||
"loss": 1.0494,
|
||||
"mean_token_accuracy": 0.7364416709169745,
|
||||
"num_tokens": 2483116.0,
|
||||
"step": 180
|
||||
},
|
||||
{
|
||||
"entropy": 1.0085792429745197,
|
||||
"epoch": 0.152,
|
||||
"grad_norm": 0.96484375,
|
||||
"learning_rate": 1.2600000000000001e-05,
|
||||
"loss": 0.9875,
|
||||
"mean_token_accuracy": 0.749194710329175,
|
||||
"num_tokens": 2617958.0,
|
||||
"step": 190
|
||||
},
|
||||
{
|
||||
"entropy": 1.000020370259881,
|
||||
"epoch": 0.16,
|
||||
"grad_norm": 0.90234375,
|
||||
"learning_rate": 1.3266666666666668e-05,
|
||||
"loss": 0.9952,
|
||||
"mean_token_accuracy": 0.7486346989870072,
|
||||
"num_tokens": 2752258.0,
|
||||
"step": 200
|
||||
},
|
||||
{
|
||||
"epoch": 0.16,
|
||||
"eval_biology_entropy": 3.2312401638031005,
|
||||
"eval_biology_loss": 3.4984476566314697,
|
||||
"eval_biology_mean_token_accuracy": 0.4106140378713608,
|
||||
"eval_biology_num_tokens": 2752258.0,
|
||||
"eval_biology_runtime": 26.9494,
|
||||
"eval_biology_samples_per_second": 18.553,
|
||||
"eval_biology_steps_per_second": 9.277,
|
||||
"step": 200
|
||||
},
|
||||
{
|
||||
"epoch": 0.16,
|
||||
"eval_chemistry_entropy": 1.9263038992881776,
|
||||
"eval_chemistry_loss": 1.9608160257339478,
|
||||
"eval_chemistry_mean_token_accuracy": 0.5982974277734756,
|
||||
"eval_chemistry_num_tokens": 2752258.0,
|
||||
"eval_chemistry_runtime": 34.4173,
|
||||
"eval_chemistry_samples_per_second": 14.528,
|
||||
"eval_chemistry_steps_per_second": 7.264,
|
||||
"step": 200
|
||||
},
|
||||
{
|
||||
"epoch": 0.16,
|
||||
"eval_math_entropy": 1.0257348996400832,
|
||||
"eval_math_loss": 1.029754638671875,
|
||||
"eval_math_mean_token_accuracy": 0.7436505243778229,
|
||||
"eval_math_num_tokens": 2752258.0,
|
||||
"eval_math_runtime": 34.9579,
|
||||
"eval_math_samples_per_second": 14.303,
|
||||
"eval_math_steps_per_second": 7.151,
|
||||
"step": 200
|
||||
},
|
||||
{
|
||||
"epoch": 0.16,
|
||||
"eval_physics_entropy": 1.5005105144977569,
|
||||
"eval_physics_loss": 1.5203113555908203,
|
||||
"eval_physics_mean_token_accuracy": 0.6669992462396622,
|
||||
"eval_physics_num_tokens": 2752258.0,
|
||||
"eval_physics_runtime": 40.5453,
|
||||
"eval_physics_samples_per_second": 12.332,
|
||||
"eval_physics_steps_per_second": 6.166,
|
||||
"step": 200
|
||||
},
|
||||
{
|
||||
"entropy": 1.0170325668528677,
|
||||
"epoch": 0.168,
|
||||
"grad_norm": 0.80859375,
|
||||
"learning_rate": 1.3933333333333334e-05,
|
||||
"loss": 1.0032,
|
||||
"mean_token_accuracy": 0.7469347029924392,
|
||||
"num_tokens": 2889249.0,
|
||||
"step": 210
|
||||
},
|
||||
{
|
||||
"entropy": 0.9709451261907815,
|
||||
"epoch": 0.176,
|
||||
"grad_norm": 0.7734375,
|
||||
"learning_rate": 1.46e-05,
|
||||
"loss": 0.9299,
|
||||
"mean_token_accuracy": 0.7556545663625002,
|
||||
"num_tokens": 3030305.0,
|
||||
"step": 220
|
||||
},
|
||||
{
|
||||
"entropy": 1.0022083457559348,
|
||||
"epoch": 0.184,
|
||||
"grad_norm": 0.90234375,
|
||||
"learning_rate": 1.5266666666666667e-05,
|
||||
"loss": 0.9815,
|
||||
"mean_token_accuracy": 0.7520246226340532,
|
||||
"num_tokens": 3167096.0,
|
||||
"step": 230
|
||||
},
|
||||
{
|
||||
"entropy": 0.9930278725922108,
|
||||
"epoch": 0.192,
|
||||
"grad_norm": 0.85546875,
|
||||
"learning_rate": 1.5933333333333336e-05,
|
||||
"loss": 0.9637,
|
||||
"mean_token_accuracy": 0.7547307416796685,
|
||||
"num_tokens": 3306293.0,
|
||||
"step": 240
|
||||
},
|
||||
{
|
||||
"entropy": 0.9768950846046209,
|
||||
"epoch": 0.2,
|
||||
"grad_norm": 0.7734375,
|
||||
"learning_rate": 1.66e-05,
|
||||
"loss": 0.9373,
|
||||
"mean_token_accuracy": 0.7594478782266378,
|
||||
"num_tokens": 3441895.0,
|
||||
"step": 250
|
||||
},
|
||||
{
|
||||
"entropy": 0.9833385257050395,
|
||||
"epoch": 0.208,
|
||||
"grad_norm": 0.8359375,
|
||||
"learning_rate": 1.726666666666667e-05,
|
||||
"loss": 0.9384,
|
||||
"mean_token_accuracy": 0.7573206882923842,
|
||||
"num_tokens": 3578349.0,
|
||||
"step": 260
|
||||
},
|
||||
{
|
||||
"entropy": 0.9361576547846198,
|
||||
"epoch": 0.216,
|
||||
"grad_norm": 0.74609375,
|
||||
"learning_rate": 1.7933333333333333e-05,
|
||||
"loss": 0.9004,
|
||||
"mean_token_accuracy": 0.7651511203497648,
|
||||
"num_tokens": 3718484.0,
|
||||
"step": 270
|
||||
},
|
||||
{
|
||||
"entropy": 0.9637048680335283,
|
||||
"epoch": 0.224,
|
||||
"grad_norm": 0.9140625,
|
||||
"learning_rate": 1.86e-05,
|
||||
"loss": 0.9349,
|
||||
"mean_token_accuracy": 0.7605184197425843,
|
||||
"num_tokens": 3851369.0,
|
||||
"step": 280
|
||||
},
|
||||
{
|
||||
"entropy": 0.9367688067257405,
|
||||
"epoch": 0.232,
|
||||
"grad_norm": 0.74609375,
|
||||
"learning_rate": 1.926666666666667e-05,
|
||||
"loss": 0.8962,
|
||||
"mean_token_accuracy": 0.7659803122282028,
|
||||
"num_tokens": 3992977.0,
|
||||
"step": 290
|
||||
},
|
||||
{
|
||||
"entropy": 0.9229113388806581,
|
||||
"epoch": 0.24,
|
||||
"grad_norm": 0.7890625,
|
||||
"learning_rate": 1.9933333333333334e-05,
|
||||
"loss": 0.9024,
|
||||
"mean_token_accuracy": 0.77176120467484,
|
||||
"num_tokens": 4133723.0,
|
||||
"step": 300
|
||||
},
|
||||
{
|
||||
"epoch": 0.24,
|
||||
"eval_biology_entropy": 3.018038348674774,
|
||||
"eval_biology_loss": 3.2969858646392822,
|
||||
"eval_biology_mean_token_accuracy": 0.43762873542308806,
|
||||
"eval_biology_num_tokens": 4133723.0,
|
||||
"eval_biology_runtime": 27.1593,
|
||||
"eval_biology_samples_per_second": 18.41,
|
||||
"eval_biology_steps_per_second": 9.205,
|
||||
"step": 300
|
||||
},
|
||||
{
|
||||
"epoch": 0.24,
|
||||
"eval_chemistry_entropy": 1.778470279932022,
|
||||
"eval_chemistry_loss": 1.8064701557159424,
|
||||
"eval_chemistry_mean_token_accuracy": 0.6220316555500031,
|
||||
"eval_chemistry_num_tokens": 4133723.0,
|
||||
"eval_chemistry_runtime": 34.6422,
|
||||
"eval_chemistry_samples_per_second": 14.433,
|
||||
"eval_chemistry_steps_per_second": 7.217,
|
||||
"step": 300
|
||||
},
|
||||
{
|
||||
"epoch": 0.24,
|
||||
"eval_math_entropy": 0.9541566734313964,
|
||||
"eval_math_loss": 0.9326547980308533,
|
||||
"eval_math_mean_token_accuracy": 0.7629979259967804,
|
||||
"eval_math_num_tokens": 4133723.0,
|
||||
"eval_math_runtime": 35.1904,
|
||||
"eval_math_samples_per_second": 14.208,
|
||||
"eval_math_steps_per_second": 7.104,
|
||||
"step": 300
|
||||
},
|
||||
{
|
||||
"epoch": 0.24,
|
||||
"eval_physics_entropy": 1.387200428724289,
|
||||
"eval_physics_loss": 1.3932181596755981,
|
||||
"eval_physics_mean_token_accuracy": 0.6886798795461655,
|
||||
"eval_physics_num_tokens": 4133723.0,
|
||||
"eval_physics_runtime": 40.7999,
|
||||
"eval_physics_samples_per_second": 12.255,
|
||||
"eval_physics_steps_per_second": 6.127,
|
||||
"step": 300
|
||||
},
|
||||
{
|
||||
"entropy": 0.9078424336388707,
|
||||
"epoch": 0.248,
|
||||
"grad_norm": 0.75,
|
||||
"learning_rate": 1.9933333333333334e-05,
|
||||
"loss": 0.8542,
|
||||
"mean_token_accuracy": 0.7745326109230518,
|
||||
"num_tokens": 4273949.0,
|
||||
"step": 310
|
||||
},
|
||||
{
|
||||
"entropy": 0.9262684918940067,
|
||||
"epoch": 0.256,
|
||||
"grad_norm": 0.84765625,
|
||||
"learning_rate": 1.985925925925926e-05,
|
||||
"loss": 0.8761,
|
||||
"mean_token_accuracy": 0.7701829686760903,
|
||||
"num_tokens": 4414102.0,
|
||||
"step": 320
|
||||
},
|
||||
{
|
||||
"entropy": 0.9303955681622028,
|
||||
"epoch": 0.264,
|
||||
"grad_norm": 0.78515625,
|
||||
"learning_rate": 1.9785185185185187e-05,
|
||||
"loss": 0.8765,
|
||||
"mean_token_accuracy": 0.7714239094406367,
|
||||
"num_tokens": 4548402.0,
|
||||
"step": 330
|
||||
},
|
||||
{
|
||||
"entropy": 0.9207985673099757,
|
||||
"epoch": 0.272,
|
||||
"grad_norm": 0.7109375,
|
||||
"learning_rate": 1.971111111111111e-05,
|
||||
"loss": 0.8728,
|
||||
"mean_token_accuracy": 0.7721738632768392,
|
||||
"num_tokens": 4685739.0,
|
||||
"step": 340
|
||||
},
|
||||
{
|
||||
"entropy": 0.9172879874706268,
|
||||
"epoch": 0.28,
|
||||
"grad_norm": 0.80078125,
|
||||
"learning_rate": 1.963703703703704e-05,
|
||||
"loss": 0.869,
|
||||
"mean_token_accuracy": 0.7719811659306288,
|
||||
"num_tokens": 4827264.0,
|
||||
"step": 350
|
||||
},
|
||||
{
|
||||
"entropy": 0.8906141923740506,
|
||||
"epoch": 0.288,
|
||||
"grad_norm": 0.76171875,
|
||||
"learning_rate": 1.9562962962962964e-05,
|
||||
"loss": 0.8676,
|
||||
"mean_token_accuracy": 0.779513431340456,
|
||||
"num_tokens": 4963900.0,
|
||||
"step": 360
|
||||
},
|
||||
{
|
||||
"entropy": 0.9126916229724884,
|
||||
"epoch": 0.296,
|
||||
"grad_norm": 0.671875,
|
||||
"learning_rate": 1.948888888888889e-05,
|
||||
"loss": 0.8727,
|
||||
"mean_token_accuracy": 0.7722058985382318,
|
||||
"num_tokens": 5106308.0,
|
||||
"step": 370
|
||||
},
|
||||
{
|
||||
"entropy": 0.9109785668551922,
|
||||
"epoch": 0.304,
|
||||
"grad_norm": 0.76171875,
|
||||
"learning_rate": 1.9414814814814817e-05,
|
||||
"loss": 0.8562,
|
||||
"mean_token_accuracy": 0.7752636637538671,
|
||||
"num_tokens": 5246290.0,
|
||||
"step": 380
|
||||
},
|
||||
{
|
||||
"entropy": 0.9183575309813022,
|
||||
"epoch": 0.312,
|
||||
"grad_norm": 0.71875,
|
||||
"learning_rate": 1.9340740740740743e-05,
|
||||
"loss": 0.8621,
|
||||
"mean_token_accuracy": 0.7713671930134296,
|
||||
"num_tokens": 5389824.0,
|
||||
"step": 390
|
||||
},
|
||||
{
|
||||
"entropy": 0.8591190708801151,
|
||||
"epoch": 0.32,
|
||||
"grad_norm": 0.82421875,
|
||||
"learning_rate": 1.926666666666667e-05,
|
||||
"loss": 0.8065,
|
||||
"mean_token_accuracy": 0.7856244947761297,
|
||||
"num_tokens": 5531902.0,
|
||||
"step": 400
|
||||
},
|
||||
{
|
||||
"epoch": 0.32,
|
||||
"eval_biology_entropy": 2.974372909069061,
|
||||
"eval_biology_loss": 3.2137467861175537,
|
||||
"eval_biology_mean_token_accuracy": 0.44984077060222627,
|
||||
"eval_biology_num_tokens": 5531902.0,
|
||||
"eval_biology_runtime": 26.9787,
|
||||
"eval_biology_samples_per_second": 18.533,
|
||||
"eval_biology_steps_per_second": 9.267,
|
||||
"step": 400
|
||||
},
|
||||
{
|
||||
"epoch": 0.32,
|
||||
"eval_chemistry_entropy": 1.7318408770561218,
|
||||
"eval_chemistry_loss": 1.746923804283142,
|
||||
"eval_chemistry_mean_token_accuracy": 0.632696654677391,
|
||||
"eval_chemistry_num_tokens": 5531902.0,
|
||||
"eval_chemistry_runtime": 34.5046,
|
||||
"eval_chemistry_samples_per_second": 14.491,
|
||||
"eval_chemistry_steps_per_second": 7.245,
|
||||
"step": 400
|
||||
},
|
||||
{
|
||||
"epoch": 0.32,
|
||||
"eval_math_entropy": 0.9170646283626557,
|
||||
"eval_math_loss": 0.8942311406135559,
|
||||
"eval_math_mean_token_accuracy": 0.7709064717292786,
|
||||
"eval_math_num_tokens": 5531902.0,
|
||||
"eval_math_runtime": 35.0734,
|
||||
"eval_math_samples_per_second": 14.256,
|
||||
"eval_math_steps_per_second": 7.128,
|
||||
"step": 400
|
||||
},
|
||||
{
|
||||
"epoch": 0.32,
|
||||
"eval_physics_entropy": 1.349606383562088,
|
||||
"eval_physics_loss": 1.343300223350525,
|
||||
"eval_physics_mean_token_accuracy": 0.6978710038661957,
|
||||
"eval_physics_num_tokens": 5531902.0,
|
||||
"eval_physics_runtime": 40.7692,
|
||||
"eval_physics_samples_per_second": 12.264,
|
||||
"eval_physics_steps_per_second": 6.132,
|
||||
"step": 400
|
||||
},
|
||||
{
|
||||
"entropy": 0.9086290217936039,
|
||||
"epoch": 0.328,
|
||||
"grad_norm": 0.796875,
|
||||
"learning_rate": 1.9192592592592593e-05,
|
||||
"loss": 0.847,
|
||||
"mean_token_accuracy": 0.7745547294616699,
|
||||
"num_tokens": 5669047.0,
|
||||
"step": 410
|
||||
},
|
||||
{
|
||||
"entropy": 0.9156009530648589,
|
||||
"epoch": 0.336,
|
||||
"grad_norm": 0.7578125,
|
||||
"learning_rate": 1.911851851851852e-05,
|
||||
"loss": 0.8769,
|
||||
"mean_token_accuracy": 0.7692474499344826,
|
||||
"num_tokens": 5806889.0,
|
||||
"step": 420
|
||||
},
|
||||
{
|
||||
"entropy": 0.8934111742302775,
|
||||
"epoch": 0.344,
|
||||
"grad_norm": 0.67578125,
|
||||
"learning_rate": 1.9044444444444446e-05,
|
||||
"loss": 0.8462,
|
||||
"mean_token_accuracy": 0.7796071924269199,
|
||||
"num_tokens": 5947019.0,
|
||||
"step": 430
|
||||
},
|
||||
{
|
||||
"entropy": 0.8856402298435568,
|
||||
"epoch": 0.352,
|
||||
"grad_norm": 0.7578125,
|
||||
"learning_rate": 1.8970370370370372e-05,
|
||||
"loss": 0.8362,
|
||||
"mean_token_accuracy": 0.7777645040303469,
|
||||
"num_tokens": 6083443.0,
|
||||
"step": 440
|
||||
},
|
||||
{
|
||||
"entropy": 0.8964365001767873,
|
||||
"epoch": 0.36,
|
||||
"grad_norm": 0.7109375,
|
||||
"learning_rate": 1.8896296296296295e-05,
|
||||
"loss": 0.8406,
|
||||
"mean_token_accuracy": 0.776063571870327,
|
||||
"num_tokens": 6215195.0,
|
||||
"step": 450
|
||||
},
|
||||
{
|
||||
"entropy": 0.8810398764908314,
|
||||
"epoch": 0.368,
|
||||
"grad_norm": 0.69921875,
|
||||
"learning_rate": 1.8822222222222225e-05,
|
||||
"loss": 0.8247,
|
||||
"mean_token_accuracy": 0.7789717759937048,
|
||||
"num_tokens": 6357305.0,
|
||||
"step": 460
|
||||
},
|
||||
{
|
||||
"entropy": 0.8900227572768926,
|
||||
"epoch": 0.376,
|
||||
"grad_norm": 0.6875,
|
||||
"learning_rate": 1.874814814814815e-05,
|
||||
"loss": 0.8424,
|
||||
"mean_token_accuracy": 0.7792754102498293,
|
||||
"num_tokens": 6498815.0,
|
||||
"step": 470
|
||||
},
|
||||
{
|
||||
"entropy": 0.8727197473868727,
|
||||
"epoch": 0.384,
|
||||
"grad_norm": 0.68359375,
|
||||
"learning_rate": 1.8674074074074075e-05,
|
||||
"loss": 0.8209,
|
||||
"mean_token_accuracy": 0.7820516306906938,
|
||||
"num_tokens": 6635429.0,
|
||||
"step": 480
|
||||
},
|
||||
{
|
||||
"entropy": 0.8815433232113719,
|
||||
"epoch": 0.392,
|
||||
"grad_norm": 0.73828125,
|
||||
"learning_rate": 1.86e-05,
|
||||
"loss": 0.8274,
|
||||
"mean_token_accuracy": 0.7792554065585137,
|
||||
"num_tokens": 6774408.0,
|
||||
"step": 490
|
||||
},
|
||||
{
|
||||
"entropy": 0.9402453400194645,
|
||||
"epoch": 0.4,
|
||||
"grad_norm": 0.78515625,
|
||||
"learning_rate": 1.8525925925925928e-05,
|
||||
"loss": 0.9268,
|
||||
"mean_token_accuracy": 0.7666260946542025,
|
||||
"num_tokens": 6914489.0,
|
||||
"step": 500
|
||||
},
|
||||
{
|
||||
"epoch": 0.4,
|
||||
"eval_biology_entropy": 2.984537806510925,
|
||||
"eval_biology_loss": 3.180291175842285,
|
||||
"eval_biology_mean_token_accuracy": 0.4551255764961243,
|
||||
"eval_biology_num_tokens": 6914489.0,
|
||||
"eval_biology_runtime": 26.9382,
|
||||
"eval_biology_samples_per_second": 18.561,
|
||||
"eval_biology_steps_per_second": 9.281,
|
||||
"step": 500
|
||||
},
|
||||
{
|
||||
"epoch": 0.4,
|
||||
"eval_chemistry_entropy": 1.735370470046997,
|
||||
"eval_chemistry_loss": 1.7186591625213623,
|
||||
"eval_chemistry_mean_token_accuracy": 0.6382190730571747,
|
||||
"eval_chemistry_num_tokens": 6914489.0,
|
||||
"eval_chemistry_runtime": 34.4276,
|
||||
"eval_chemistry_samples_per_second": 14.523,
|
||||
"eval_chemistry_steps_per_second": 7.262,
|
||||
"step": 500
|
||||
},
|
||||
{
|
||||
"epoch": 0.4,
|
||||
"eval_math_entropy": 0.9168894953727722,
|
||||
"eval_math_loss": 0.8749539852142334,
|
||||
"eval_math_mean_token_accuracy": 0.7746825203895569,
|
||||
"eval_math_num_tokens": 6914489.0,
|
||||
"eval_math_runtime": 34.9026,
|
||||
"eval_math_samples_per_second": 14.326,
|
||||
"eval_math_steps_per_second": 7.163,
|
||||
"step": 500
|
||||
},
|
||||
{
|
||||
"epoch": 0.4,
|
||||
"eval_physics_entropy": 1.34889164853096,
|
||||
"eval_physics_loss": 1.3155016899108887,
|
||||
"eval_physics_mean_token_accuracy": 0.7029694278240204,
|
||||
"eval_physics_num_tokens": 6914489.0,
|
||||
"eval_physics_runtime": 40.5298,
|
||||
"eval_physics_samples_per_second": 12.337,
|
||||
"eval_physics_steps_per_second": 6.168,
|
||||
"step": 500
|
||||
}
|
||||
],
|
||||
"logging_steps": 10,
|
||||
"max_steps": 3000,
|
||||
"num_input_tokens_seen": 0,
|
||||
"num_train_epochs": 3,
|
||||
"save_steps": 500,
|
||||
"stateful_callbacks": {
|
||||
"TrainerControl": {
|
||||
"args": {
|
||||
"should_epoch_stop": false,
|
||||
"should_evaluate": false,
|
||||
"should_log": false,
|
||||
"should_save": true,
|
||||
"should_training_stop": false
|
||||
},
|
||||
"attributes": {}
|
||||
}
|
||||
},
|
||||
"total_flos": 3.630158407358546e+17,
|
||||
"train_batch_size": 2,
|
||||
"trial_name": null,
|
||||
"trial_params": null
|
||||
}
|
||||
3
checkpoint-500/training_args.bin
Normal file
3
checkpoint-500/training_args.bin
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:468bf7cb033f721ea40664a43923a7a349b3c291ee5b69f9acfdcec3e7250e85
|
||||
size 6481
|
||||
1
checkpoint-500/vocab.json
Normal file
1
checkpoint-500/vocab.json
Normal file
File diff suppressed because one or more lines are too long
Reference in New Issue
Block a user