初始化项目，由ModelHub XC社区提供模型
Model: KKHYA/qwen3-1.7b-fft-coding Source: Original Platform
This commit is contained in:
ModelHub XC
36
.gitattributes
vendored
Normal file
36
.gitattributes
vendored
Normal file
@@ -0,0 +1,36 @@
|
|||||||
|
*.7z filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.arrow filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.bin filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.bz2 filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.ckpt filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.ftz filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.gz filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.h5 filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.joblib filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.lfs.* filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.mlmodel filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.model filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.msgpack filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.npy filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.npz filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.onnx filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.ot filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.parquet filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.pb filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.pickle filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.pkl filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.pt filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.pth filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.rar filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.safetensors filter=lfs diff=lfs merge=lfs -text
|
||||||
|
saved_model/**/* filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.tar.* filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.tar filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.tflite filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.tgz filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.wasm filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.xz filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.zip filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.zst filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
||||||
|
tokenizer.json filter=lfs diff=lfs merge=lfs -text
|
||||||
61
README.md
Normal file
61
README.md
Normal file
@@ -0,0 +1,61 @@
|
|||||||
|
---
|
||||||
|
library_name: transformers
|
||||||
|
license: apache-2.0
|
||||||
|
base_model: Qwen/Qwen3-1.7B
|
||||||
|
tags:
|
||||||
|
- llama-factory
|
||||||
|
- full
|
||||||
|
- generated_from_trainer
|
||||||
|
model-index:
|
||||||
|
- name: qwen3-1.7b-fft-coding
|
||||||
|
results: []
|
||||||
|
---
|
||||||
|
|
||||||
|
<!-- This model card has been generated automatically according to the information the Trainer had access to. You
|
||||||
|
should probably proofread and complete it, then remove this comment. -->
|
||||||
|
|
||||||
|
# qwen3-1.7b-fft-coding
|
||||||
|
|
||||||
|
This model is a fine-tuned version of [Qwen/Qwen3-1.7B](https://huggingface.co/Qwen/Qwen3-1.7B) on the mft_tulu3_personas_code, the mft_evol_codealpaca and the mft_codefeedback datasets.
|
||||||
|
|
||||||
|
## Model description
|
||||||
|
|
||||||
|
More information needed
|
||||||
|
|
||||||
|
## Intended uses & limitations
|
||||||
|
|
||||||
|
More information needed
|
||||||
|
|
||||||
|
## Training and evaluation data
|
||||||
|
|
||||||
|
More information needed
|
||||||
|
|
||||||
|
## Training procedure
|
||||||
|
|
||||||
|
### Training hyperparameters
|
||||||
|
|
||||||
|
The following hyperparameters were used during training:
|
||||||
|
- learning_rate: 1e-05
|
||||||
|
- train_batch_size: 2
|
||||||
|
- eval_batch_size: 8
|
||||||
|
- seed: 42
|
||||||
|
- distributed_type: multi-GPU
|
||||||
|
- num_devices: 8
|
||||||
|
- gradient_accumulation_steps: 8
|
||||||
|
- total_train_batch_size: 128
|
||||||
|
- total_eval_batch_size: 64
|
||||||
|
- optimizer: Use OptimizerNames.ADAMW_TORCH_FUSED with betas=(0.9,0.95) and epsilon=1e-08 and optimizer_args=No additional optimizer arguments
|
||||||
|
- lr_scheduler_type: cosine
|
||||||
|
- lr_scheduler_warmup_steps: 0.1
|
||||||
|
- num_epochs: 4.0
|
||||||
|
|
||||||
|
### Training results
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
### Framework versions
|
||||||
|
|
||||||
|
- Transformers 5.2.0
|
||||||
|
- Pytorch 2.10.0+cu130
|
||||||
|
- Datasets 4.0.0
|
||||||
|
- Tokenizers 0.22.2
|
||||||
8
all_results.json
Normal file
8
all_results.json
Normal file
@@ -0,0 +1,8 @@
|
|||||||
|
{
|
||||||
|
"epoch": 4.0,
|
||||||
|
"total_flos": 7.974893111623025e+17,
|
||||||
|
"train_loss": 0.6604051219656112,
|
||||||
|
"train_runtime": 3155.399,
|
||||||
|
"train_samples_per_second": 38.03,
|
||||||
|
"train_steps_per_second": 0.298
|
||||||
|
}
|
||||||
89
chat_template.jinja
Normal file
89
chat_template.jinja
Normal file
@@ -0,0 +1,89 @@
|
|||||||
|
{%- if tools %}
|
||||||
|
{{- '<|im_start|>system\n' }}
|
||||||
|
{%- if messages[0].role == 'system' %}
|
||||||
|
{{- messages[0].content + '\n\n' }}
|
||||||
|
{%- endif %}
|
||||||
|
{{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within <tools></tools> XML tags:\n<tools>" }}
|
||||||
|
{%- for tool in tools %}
|
||||||
|
{{- "\n" }}
|
||||||
|
{{- tool | tojson }}
|
||||||
|
{%- endfor %}
|
||||||
|
{{- "\n</tools>\n\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\n<tool_call>\n{\"name\": <function-name>, \"arguments\": <args-json-object>}\n</tool_call><|im_end|>\n" }}
|
||||||
|
{%- else %}
|
||||||
|
{%- if messages[0].role == 'system' %}
|
||||||
|
{{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }}
|
||||||
|
{%- endif %}
|
||||||
|
{%- endif %}
|
||||||
|
{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %}
|
||||||
|
{%- for message in messages[::-1] %}
|
||||||
|
{%- set index = (messages|length - 1) - loop.index0 %}
|
||||||
|
{%- if ns.multi_step_tool and message.role == "user" and message.content is string and not(message.content.startswith('<tool_response>') and message.content.endswith('</tool_response>')) %}
|
||||||
|
{%- set ns.multi_step_tool = false %}
|
||||||
|
{%- set ns.last_query_index = index %}
|
||||||
|
{%- endif %}
|
||||||
|
{%- endfor %}
|
||||||
|
{%- for message in messages %}
|
||||||
|
{%- if message.content is string %}
|
||||||
|
{%- set content = message.content %}
|
||||||
|
{%- else %}
|
||||||
|
{%- set content = '' %}
|
||||||
|
{%- endif %}
|
||||||
|
{%- if (message.role == "user") or (message.role == "system" and not loop.first) %}
|
||||||
|
{{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }}
|
||||||
|
{%- elif message.role == "assistant" %}
|
||||||
|
{%- set reasoning_content = '' %}
|
||||||
|
{%- if message.reasoning_content is string %}
|
||||||
|
{%- set reasoning_content = message.reasoning_content %}
|
||||||
|
{%- else %}
|
||||||
|
{%- if '</think>' in content %}
|
||||||
|
{%- set reasoning_content = content.split('</think>')[0].rstrip('\n').split('<think>')[-1].lstrip('\n') %}
|
||||||
|
{%- set content = content.split('</think>')[-1].lstrip('\n') %}
|
||||||
|
{%- endif %}
|
||||||
|
{%- endif %}
|
||||||
|
{%- if loop.index0 > ns.last_query_index %}
|
||||||
|
{%- if loop.last or (not loop.last and reasoning_content) %}
|
||||||
|
{{- '<|im_start|>' + message.role + '\n<think>\n' + reasoning_content.strip('\n') + '\n</think>\n\n' + content.lstrip('\n') }}
|
||||||
|
{%- else %}
|
||||||
|
{{- '<|im_start|>' + message.role + '\n' + content }}
|
||||||
|
{%- endif %}
|
||||||
|
{%- else %}
|
||||||
|
{{- '<|im_start|>' + message.role + '\n' + content }}
|
||||||
|
{%- endif %}
|
||||||
|
{%- if message.tool_calls %}
|
||||||
|
{%- for tool_call in message.tool_calls %}
|
||||||
|
{%- if (loop.first and content) or (not loop.first) %}
|
||||||
|
{{- '\n' }}
|
||||||
|
{%- endif %}
|
||||||
|
{%- if tool_call.function %}
|
||||||
|
{%- set tool_call = tool_call.function %}
|
||||||
|
{%- endif %}
|
||||||
|
{{- '<tool_call>\n{"name": "' }}
|
||||||
|
{{- tool_call.name }}
|
||||||
|
{{- '", "arguments": ' }}
|
||||||
|
{%- if tool_call.arguments is string %}
|
||||||
|
{{- tool_call.arguments }}
|
||||||
|
{%- else %}
|
||||||
|
{{- tool_call.arguments | tojson }}
|
||||||
|
{%- endif %}
|
||||||
|
{{- '}\n</tool_call>' }}
|
||||||
|
{%- endfor %}
|
||||||
|
{%- endif %}
|
||||||
|
{{- '<|im_end|>\n' }}
|
||||||
|
{%- elif message.role == "tool" %}
|
||||||
|
{%- if loop.first or (messages[loop.index0 - 1].role != "tool") %}
|
||||||
|
{{- '<|im_start|>user' }}
|
||||||
|
{%- endif %}
|
||||||
|
{{- '\n<tool_response>\n' }}
|
||||||
|
{{- content }}
|
||||||
|
{{- '\n</tool_response>' }}
|
||||||
|
{%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
|
||||||
|
{{- '<|im_end|>\n' }}
|
||||||
|
{%- endif %}
|
||||||
|
{%- endif %}
|
||||||
|
{%- endfor %}
|
||||||
|
{%- if add_generation_prompt %}
|
||||||
|
{{- '<|im_start|>assistant\n' }}
|
||||||
|
{%- if enable_thinking is defined and enable_thinking is false %}
|
||||||
|
{{- '<think>\n\n</think>\n\n' }}
|
||||||
|
{%- endif %}
|
||||||
|
{%- endif %}
|
||||||
70
config.json
Normal file
70
config.json
Normal file
@@ -0,0 +1,70 @@
|
|||||||
|
{
|
||||||
|
"architectures": [
|
||||||
|
"Qwen3ForCausalLM"
|
||||||
|
],
|
||||||
|
"attention_bias": false,
|
||||||
|
"attention_dropout": 0.0,
|
||||||
|
"bos_token_id": null,
|
||||||
|
"dtype": "bfloat16",
|
||||||
|
"eos_token_id": 151645,
|
||||||
|
"head_dim": 128,
|
||||||
|
"hidden_act": "silu",
|
||||||
|
"hidden_size": 2048,
|
||||||
|
"initializer_range": 0.02,
|
||||||
|
"intermediate_size": 6144,
|
||||||
|
"layer_types": [
|
||||||
|
"full_attention",
|
||||||
|
"full_attention",
|
||||||
|
"full_attention",
|
||||||
|
"full_attention",
|
||||||
|
"full_attention",
|
||||||
|
"full_attention",
|
||||||
|
"full_attention",
|
||||||
|
"full_attention",
|
||||||
|
"full_attention",
|
||||||
|
"full_attention",
|
||||||
|
"full_attention",
|
||||||
|
"full_attention",
|
||||||
|
"full_attention",
|
||||||
|
"full_attention",
|
||||||
|
"full_attention",
|
||||||
|
"full_attention",
|
||||||
|
"full_attention",
|
||||||
|
"full_attention",
|
||||||
|
"full_attention",
|
||||||
|
"full_attention",
|
||||||
|
"full_attention",
|
||||||
|
"full_attention",
|
||||||
|
"full_attention",
|
||||||
|
"full_attention",
|
||||||
|
"full_attention",
|
||||||
|
"full_attention",
|
||||||
|
"full_attention",
|
||||||
|
"full_attention"
|
||||||
|
],
|
||||||
|
"masked_layers": null,
|
||||||
|
"max_position_embeddings": 40960,
|
||||||
|
"max_window_layers": 28,
|
||||||
|
"model_type": "qwen3",
|
||||||
|
"num_attention_heads": 16,
|
||||||
|
"num_hidden_layers": 28,
|
||||||
|
"num_key_value_heads": 8,
|
||||||
|
"pad_token_id": 151643,
|
||||||
|
"rms_norm_eps": 1e-06,
|
||||||
|
"rope_parameters": {
|
||||||
|
"rope_theta": 1000000,
|
||||||
|
"rope_type": "default"
|
||||||
|
},
|
||||||
|
"sliding_window": null,
|
||||||
|
"sparsity_attn": null,
|
||||||
|
"sparsity_mlp": null,
|
||||||
|
"subnet_mode": null,
|
||||||
|
"subnet_type": null,
|
||||||
|
"threshold_attn": null,
|
||||||
|
"threshold_mlp": null,
|
||||||
|
"tie_word_embeddings": true,
|
||||||
|
"transformers_version": "5.2.0",
|
||||||
|
"use_cache": false,
|
||||||
|
"use_sliding_window": false,
|
||||||
|
"vocab_size": 151936
|
||||||
|
}
|
||||||
12
generation_config.json
Normal file
12
generation_config.json
Normal file
@@ -0,0 +1,12 @@
|
|||||||
|
{
|
||||||
|
"do_sample": true,
|
||||||
|
"eos_token_id": [
|
||||||
|
151645,
|
||||||
|
151643
|
||||||
|
],
|
||||||
|
"pad_token_id": 151643,
|
||||||
|
"temperature": 0.6,
|
||||||
|
"top_k": 20,
|
||||||
|
"top_p": 0.95,
|
||||||
|
"transformers_version": "5.2.0"
|
||||||
|
}
|
||||||
3
model.safetensors
Normal file
3
model.safetensors
Normal file
@@ -0,0 +1,3 @@
|
|||||||
|
version https://git-lfs.github.com/spec/v1
|
||||||
|
oid sha256:95bacf21dfe9f54063f1304cbace1030b135858e35f2478310f0c87a17c3467d
|
||||||
|
size 4063515640
|
||||||
219
qwen3_1.7b_coding_fft_20260429_153648.log
Normal file
219
qwen3_1.7b_coding_fft_20260429_153648.log
Normal file
@@ -0,0 +1,219 @@
|
|||||||
|
==== STARTING EXPERIMENT: qwen3_1.7b_coding_fft ====
|
||||||
|
Log File: saves/qwen3_1.7b/coding/fft/qwen3_1.7b_coding_fft_20260429_153648.log
|
||||||
|
HF Hub: https://huggingface.co/KKHYA/qwen3-1.7b-fft-coding
|
||||||
|
Timestamp: 2026-04-29 15:36:48
|
||||||
|
=====================================
|
||||||
|
[INFO|2026-04-29 15:37:02] llamafactory.launcher:144 >> Initializing 8 distributed tasks at: 127.0.0.1:42841
|
||||||
|
W0429 15:37:04.319000 212778 site-packages/torch/distributed/run.py:852]
|
||||||
|
W0429 15:37:04.319000 212778 site-packages/torch/distributed/run.py:852] *****************************************
|
||||||
|
W0429 15:37:04.319000 212778 site-packages/torch/distributed/run.py:852] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed.
|
||||||
|
W0429 15:37:04.319000 212778 site-packages/torch/distributed/run.py:852] *****************************************
|
||||||
|
warmup_ratio is deprecated and will be removed in v5.2. Use `warmup_steps` instead.
|
||||||
|
warmup_ratio is deprecated and will be removed in v5.2. Use `warmup_steps` instead.
|
||||||
|
warmup_ratio is deprecated and will be removed in v5.2. Use `warmup_steps` instead.
|
||||||
|
df: /root/.triton/autotune: No such file or directory
|
||||||
|
warmup_ratio is deprecated and will be removed in v5.2. Use `warmup_steps` instead.
|
||||||
|
warmup_ratio is deprecated and will be removed in v5.2. Use `warmup_steps` instead.
|
||||||
|
warmup_ratio is deprecated and will be removed in v5.2. Use `warmup_steps` instead.
|
||||||
|
warmup_ratio is deprecated and will be removed in v5.2. Use `warmup_steps` instead.
|
||||||
|
warmup_ratio is deprecated and will be removed in v5.2. Use `warmup_steps` instead.
|
||||||
|
[W429 15:37:18.649985073 ProcessGroupNCCL.cpp:929] Warning: TORCH_NCCL_AVOID_RECORD_STREAMS is the default now, this environment variable is thus deprecated. (function operator())
|
||||||
|
[W429 15:37:18.746117814 ProcessGroupNCCL.cpp:929] Warning: TORCH_NCCL_AVOID_RECORD_STREAMS is the default now, this environment variable is thus deprecated. (function operator())
|
||||||
|
[W429 15:37:18.778768535 ProcessGroupNCCL.cpp:929] Warning: TORCH_NCCL_AVOID_RECORD_STREAMS is the default now, this environment variable is thus deprecated. (function operator())
|
||||||
|
[W429 15:37:18.828110308 ProcessGroupNCCL.cpp:929] Warning: TORCH_NCCL_AVOID_RECORD_STREAMS is the default now, this environment variable is thus deprecated. (function operator())
|
||||||
|
[W429 15:37:18.868414499 ProcessGroupNCCL.cpp:929] Warning: TORCH_NCCL_AVOID_RECORD_STREAMS is the default now, this environment variable is thus deprecated. (function operator())
|
||||||
|
[W429 15:37:18.947969051 ProcessGroupNCCL.cpp:929] Warning: TORCH_NCCL_AVOID_RECORD_STREAMS is the default now, this environment variable is thus deprecated. (function operator())
|
||||||
|
ywang29-p4d-debug-1-worker-0:212861:212861 [0] NCCL INFO ENV/Plugin: Could not find: libnccl-env.so
|
||||||
|
ywang29-p4d-debug-1-worker-0:212861:212861 [0] NCCL INFO NCCL_SOCKET_IFNAME set by environment to ^docker0,lo
|
||||||
|
ywang29-p4d-debug-1-worker-0:212861:212861 [0] NCCL INFO Bootstrap: Using eth0:10.200.170.96<0>
|
||||||
|
ywang29-p4d-debug-1-worker-0:212861:212861 [0] NCCL INFO cudaDriverVersion 13000
|
||||||
|
ywang29-p4d-debug-1-worker-0:212862:212862 [1] NCCL INFO ENV/Plugin: Could not find: libnccl-env.so
|
||||||
|
ywang29-p4d-debug-1-worker-0:212862:212862 [1] NCCL INFO cudaDriverVersion 13000
|
||||||
|
ywang29-p4d-debug-1-worker-0:212862:212862 [1] NCCL INFO NCCL_SOCKET_IFNAME set by environment to ^docker0,lo
|
||||||
|
ywang29-p4d-debug-1-worker-0:212862:212862 [1] NCCL INFO Bootstrap: Using eth0:10.200.170.96<0>
|
||||||
|
ywang29-p4d-debug-1-worker-0:212862:212862 [1] NCCL INFO NCCL version 2.28.9+cuda13.0
|
||||||
|
ywang29-p4d-debug-1-worker-0:212861:212861 [0] NCCL INFO NCCL version 2.28.9+cuda13.0
|
||||||
|
ywang29-p4d-debug-1-worker-0:212862:212862 [1] NCCL INFO Comm config Blocking set to 1
|
||||||
|
[W429 15:37:18.174754892 ProcessGroupNCCL.cpp:929] Warning: TORCH_NCCL_AVOID_RECORD_STREAMS is the default now, this environment variable is thus deprecated. (function operator())
|
||||||
|
ywang29-p4d-debug-1-worker-0:212863:212863 [2] NCCL INFO ENV/Plugin: Could not find: libnccl-env.so
|
||||||
|
ywang29-p4d-debug-1-worker-0:212863:212863 [2] NCCL INFO cudaDriverVersion 13000
|
||||||
|
ywang29-p4d-debug-1-worker-0:212863:212863 [2] NCCL INFO NCCL_SOCKET_IFNAME set by environment to ^docker0,lo
|
||||||
|
ywang29-p4d-debug-1-worker-0:212863:212863 [2] NCCL INFO Bootstrap: Using eth0:10.200.170.96<0>
|
||||||
|
ywang29-p4d-debug-1-worker-0:212863:212863 [2] NCCL INFO NCCL version 2.28.9+cuda13.0
|
||||||
|
ywang29-p4d-debug-1-worker-0:212863:212863 [2] NCCL INFO Comm config Blocking set to 1
|
||||||
|
ywang29-p4d-debug-1-worker-0:212867:212867 [6] NCCL INFO ENV/Plugin: Could not find: libnccl-env.so
|
||||||
|
ywang29-p4d-debug-1-worker-0:212867:212867 [6] NCCL INFO cudaDriverVersion 13000
|
||||||
|
ywang29-p4d-debug-1-worker-0:212862:212862 [1] NCCL INFO NET/Plugin: Loaded net plugin Libfabric (v10)
|
||||||
|
ywang29-p4d-debug-1-worker-0:212862:212862 [1] NCCL INFO Successfully loaded external network plugin /opt/amazon/ofi-nccl/lib/libnccl-net.so
|
||||||
|
ywang29-p4d-debug-1-worker-0:212862:212862 [1] NCCL INFO NET/OFI Initializing aws-ofi-nccl 1.17.1
|
||||||
|
ywang29-p4d-debug-1-worker-0:212862:212862 [1] NCCL INFO NET/OFI Using Libfabric version 2.3
|
||||||
|
ywang29-p4d-debug-1-worker-0:212862:212862 [1] NCCL INFO NET/OFI Using CUDA driver version 13000 with runtime 13000
|
||||||
|
ywang29-p4d-debug-1-worker-0:212862:212862 [1] NCCL INFO NET/OFI Configuring AWS-specific options
|
||||||
|
ywang29-p4d-debug-1-worker-0:212862:212862 [1] NCCL INFO NET/OFI Setting provider_filter to efa
|
||||||
|
ywang29-p4d-debug-1-worker-0:212862:212862 [1] NCCL INFO NET/OFI Running on p4de.24xlarge platform, topology file /opt/amazon/ofi-nccl/share/aws-ofi-nccl/xml/p4de-24xl-topo.xml
|
||||||
|
ywang29-p4d-debug-1-worker-0:212862:212862 [1] NCCL INFO NET/OFI Internode latency set at 75.0 us
|
||||||
|
ywang29-p4d-debug-1-worker-0:212862:212862 [1] NCCL INFO NET/OFI Using transport protocol SENDRECV (platform set)
|
||||||
|
ywang29-p4d-debug-1-worker-0:212867:212867 [6] NCCL INFO NCCL_SOCKET_IFNAME set by environment to ^docker0,lo
|
||||||
|
ywang29-p4d-debug-1-worker-0:212867:212867 [6] NCCL INFO Bootstrap: Using eth0:10.200.170.96<0>
|
||||||
|
ywang29-p4d-debug-1-worker-0:212867:212867 [6] NCCL INFO NCCL version 2.28.9+cuda13.0
|
||||||
|
|
||||||
|
[2026-04-29 15:37:18] ywang29-p4d-debug-1-worker-0:212862:212862 [1] int nccl_net_ofi_create_plugin(nccl_net_ofi_plugin_t**):208 NCCL WARN NET/OFI Failed to initialize sendrecv protocol
|
||||||
|
|
||||||
|
[2026-04-29 15:37:18] ywang29-p4d-debug-1-worker-0:212862:212862 [1] int nccl_net_ofi_create_plugin(nccl_net_ofi_plugin_t**):353 NCCL WARN NET/OFI aws-ofi-nccl initialization failed
|
||||||
|
|
||||||
|
[2026-04-29 15:37:18] ywang29-p4d-debug-1-worker-0:212862:212862 [1] ncclResult_t nccl_net_ofi_init_v6(ncclDebugLogger_t):162 NCCL WARN NET/OFI Initializing plugin failed
|
||||||
|
ywang29-p4d-debug-1-worker-0:212862:212862 [1] NCCL INFO plugin/net/net_v10.cc:81 -> 2
|
||||||
|
ywang29-p4d-debug-1-worker-0:212862:212862 [1] NCCL INFO Failed to initialize NET plugin Libfabric
|
||||||
|
ywang29-p4d-debug-1-worker-0:212867:212867 [6] NCCL INFO Comm config Blocking set to 1
|
||||||
|
[W429 15:37:18.265962460 ProcessGroupNCCL.cpp:929] Warning: TORCH_NCCL_AVOID_RECORD_STREAMS is the default now, this environment variable is thus deprecated. (function operator())
|
||||||
|
ywang29-p4d-debug-1-worker-0:212865:212865 [4] NCCL INFO ENV/Plugin: Could not find: libnccl-env.so
|
||||||
|
ywang29-p4d-debug-1-worker-0:212865:212865 [4] NCCL INFO cudaDriverVersion 13000
|
||||||
|
ywang29-p4d-debug-1-worker-0:212865:212865 [4] NCCL INFO NCCL_SOCKET_IFNAME set by environment to ^docker0,lo
|
||||||
|
ywang29-p4d-debug-1-worker-0:212865:212865 [4] NCCL INFO Bootstrap: Using eth0:10.200.170.96<0>
|
||||||
|
ywang29-p4d-debug-1-worker-0:212865:212865 [4] NCCL INFO NCCL version 2.28.9+cuda13.0
|
||||||
|
ywang29-p4d-debug-1-worker-0:212865:212865 [4] NCCL INFO Comm config Blocking set to 1
|
||||||
|
ywang29-p4d-debug-1-worker-0:212868:212868 [7] NCCL INFO ENV/Plugin: Could not find: libnccl-env.so
|
||||||
|
ywang29-p4d-debug-1-worker-0:212868:212868 [7] NCCL INFO cudaDriverVersion 13000
|
||||||
|
ywang29-p4d-debug-1-worker-0:212868:212868 [7] NCCL INFO NCCL_SOCKET_IFNAME set by environment to ^docker0,lo
|
||||||
|
ywang29-p4d-debug-1-worker-0:212868:212868 [7] NCCL INFO Bootstrap: Using eth0:10.200.170.96<0>
|
||||||
|
ywang29-p4d-debug-1-worker-0:212868:212868 [7] NCCL INFO NCCL version 2.28.9+cuda13.0
|
||||||
|
ywang29-p4d-debug-1-worker-0:212868:212868 [7] NCCL INFO Comm config Blocking set to 1
|
||||||
|
ywang29-p4d-debug-1-worker-0:212861:212861 [0] NCCL INFO Comm config Blocking set to 1
|
||||||
|
ywang29-p4d-debug-1-worker-0:212863:212863 [2] NCCL INFO NET/Plugin: Loaded net plugin Libfabric (v10)
|
||||||
|
ywang29-p4d-debug-1-worker-0:212863:212863 [2] NCCL INFO Successfully loaded external network plugin /opt/amazon/ofi-nccl/lib/libnccl-net.so
|
||||||
|
ywang29-p4d-debug-1-worker-0:212863:212863 [2] NCCL INFO NET/OFI Initializing aws-ofi-nccl 1.17.1
|
||||||
|
ywang29-p4d-debug-1-worker-0:212863:212863 [2] NCCL INFO NET/OFI Using Libfabric version 2.3
|
||||||
|
ywang29-p4d-debug-1-worker-0:212863:212863 [2] NCCL INFO NET/OFI Using CUDA driver version 13000 with runtime 13000
|
||||||
|
ywang29-p4d-debug-1-worker-0:212863:212863 [2] NCCL INFO NET/OFI Configuring AWS-specific options
|
||||||
|
ywang29-p4d-debug-1-worker-0:212863:212863 [2] NCCL INFO NET/OFI Setting provider_filter to efa
|
||||||
|
ywang29-p4d-debug-1-worker-0:212863:212863 [2] NCCL INFO NET/OFI Running on p4de.24xlarge platform, topology file /opt/amazon/ofi-nccl/share/aws-ofi-nccl/xml/p4de-24xl-topo.xml
|
||||||
|
ywang29-p4d-debug-1-worker-0:212863:212863 [2] NCCL INFO NET/OFI Internode latency set at 75.0 us
|
||||||
|
ywang29-p4d-debug-1-worker-0:212863:212863 [2] NCCL INFO NET/OFI Using transport protocol SENDRECV (platform set)
|
||||||
|
ywang29-p4d-debug-1-worker-0:212864:212864 [3] NCCL INFO ENV/Plugin: Could not find: libnccl-env.so
|
||||||
|
ywang29-p4d-debug-1-worker-0:212864:212864 [3] NCCL INFO cudaDriverVersion 13000
|
||||||
|
ywang29-p4d-debug-1-worker-0:212864:212864 [3] NCCL INFO NCCL_SOCKET_IFNAME set by environment to ^docker0,lo
|
||||||
|
|
||||||
|
[2026-04-29 15:37:18] ywang29-p4d-debug-1-worker-0:212863:212863 [2] int nccl_net_ofi_create_plugin(nccl_net_ofi_plugin_t**):208 NCCL WARN NET/OFI Failed to initialize sendrecv protocol
|
||||||
|
ywang29-p4d-debug-1-worker-0:212864:212864 [3] NCCL INFO Bootstrap: Using eth0:10.200.170.96<0>
|
||||||
|
|
||||||
|
[2026-04-29 15:37:18] ywang29-p4d-debug-1-worker-0:212863:212863 [2] int nccl_net_ofi_create_plugin(nccl_net_ofi_plugin_t**):353 NCCL WARN NET/OFI aws-ofi-nccl initialization failed
|
||||||
|
|
||||||
|
[2026-04-29 15:37:18] ywang29-p4d-debug-1-worker-0:212863:212863 [2] ncclResult_t nccl_net_ofi_init_v6(ncclDebugLogger_t):162 NCCL WARN NET/OFI Initializing plugin failed
|
||||||
|
ywang29-p4d-debug-1-worker-0:212863:212863 [2] NCCL INFO plugin/net/net_v10.cc:81 -> 2
|
||||||
|
ywang29-p4d-debug-1-worker-0:212864:212864 [3] NCCL INFO NCCL version 2.28.9+cuda13.0
|
||||||
|
ywang29-p4d-debug-1-worker-0:212863:212863 [2] NCCL INFO Failed to initialize NET plugin Libfabric
|
||||||
|
ywang29-p4d-debug-1-worker-0:212867:212867 [6] NCCL INFO NET/Plugin: Loaded net plugin Libfabric (v10)
|
||||||
|
ywang29-p4d-debug-1-worker-0:212867:212867 [6] NCCL INFO Successfully loaded external network plugin /opt/amazon/ofi-nccl/lib/libnccl-net.so
|
||||||
|
ywang29-p4d-debug-1-worker-0:212867:212867 [6] NCCL INFO NET/OFI Initializing aws-ofi-nccl 1.17.1
|
||||||
|
ywang29-p4d-debug-1-worker-0:212867:212867 [6] NCCL INFO NET/OFI Using Libfabric version 2.3
|
||||||
|
ywang29-p4d-debug-1-worker-0:212864:212864 [3] NCCL INFO Comm config Blocking set to 1
|
||||||
|
ywang29-p4d-debug-1-worker-0:212867:212867 [6] NCCL INFO NET/OFI Using CUDA driver version 13000 with runtime 13000
|
||||||
|
ywang29-p4d-debug-1-worker-0:212867:212867 [6] NCCL INFO NET/OFI Configuring AWS-specific options
|
||||||
|
ywang29-p4d-debug-1-worker-0:212867:212867 [6] NCCL INFO NET/OFI Setting provider_filter to efa
|
||||||
|
ywang29-p4d-debug-1-worker-0:212867:212867 [6] NCCL INFO NET/OFI Running on p4de.24xlarge platform, topology file /opt/amazon/ofi-nccl/share/aws-ofi-nccl/xml/p4de-24xl-topo.xml
|
||||||
|
ywang29-p4d-debug-1-worker-0:212867:212867 [6] NCCL INFO NET/OFI Internode latency set at 75.0 us
|
||||||
|
ywang29-p4d-debug-1-worker-0:212867:212867 [6] NCCL INFO NET/OFI Using transport protocol SENDRECV (platform set)
|
||||||
|
|
||||||
|
[2026-04-29 15:37:18] ywang29-p4d-debug-1-worker-0:212867:212867 [6] int nccl_net_ofi_create_plugin(nccl_net_ofi_plugin_t**):208 NCCL WARN NET/OFI Failed to initialize sendrecv protocol
|
||||||
|
|
||||||
|
[2026-04-29 15:37:18] ywang29-p4d-debug-1-worker-0:212867:212867 [6] int nccl_net_ofi_create_plugin(nccl_net_ofi_plugin_t**):353 NCCL WARN NET/OFI aws-ofi-nccl initialization failed
|
||||||
|
|
||||||
|
[2026-04-29 15:37:18] ywang29-p4d-debug-1-worker-0:212867:212867 [6] ncclResult_t nccl_net_ofi_init_v6(ncclDebugLogger_t):162 NCCL WARN NET/OFI Initializing plugin failed
|
||||||
|
ywang29-p4d-debug-1-worker-0:212867:212867 [6] NCCL INFO plugin/net/net_v10.cc:81 -> 2
|
||||||
|
ywang29-p4d-debug-1-worker-0:212867:212867 [6] NCCL INFO Failed to initialize NET plugin Libfabric
|
||||||
|
ywang29-p4d-debug-1-worker-0:212866:212866 [5] NCCL INFO ENV/Plugin: Could not find: libnccl-env.so
|
||||||
|
ywang29-p4d-debug-1-worker-0:212866:212866 [5] NCCL INFO cudaDriverVersion 13000
|
||||||
|
ywang29-p4d-debug-1-worker-0:212866:212866 [5] NCCL INFO NCCL_SOCKET_IFNAME set by environment to ^docker0,lo
|
||||||
|
ywang29-p4d-debug-1-worker-0:212866:212866 [5] NCCL INFO Bootstrap: Using eth0:10.200.170.96<0>
|
||||||
|
ywang29-p4d-debug-1-worker-0:212866:212866 [5] NCCL INFO NCCL version 2.28.9+cuda13.0
|
||||||
|
ywang29-p4d-debug-1-worker-0:212866:212866 [5] NCCL INFO Comm config Blocking set to 1
|
||||||
|
ywang29-p4d-debug-1-worker-0:212865:212865 [4] NCCL INFO NET/Plugin: Loaded net plugin Libfabric (v10)
|
||||||
|
ywang29-p4d-debug-1-worker-0:212865:212865 [4] NCCL INFO Successfully loaded external network plugin /opt/amazon/ofi-nccl/lib/libnccl-net.so
|
||||||
|
ywang29-p4d-debug-1-worker-0:212865:212865 [4] NCCL INFO NET/OFI Initializing aws-ofi-nccl 1.17.1
|
||||||
|
ywang29-p4d-debug-1-worker-0:212865:212865 [4] NCCL INFO NET/OFI Using Libfabric version 2.3
|
||||||
|
ywang29-p4d-debug-1-worker-0:212865:212865 [4] NCCL INFO NET/OFI Using CUDA driver version 13000 with runtime 13000
|
||||||
|
ywang29-p4d-debug-1-worker-0:212865:212865 [4] NCCL INFO NET/OFI Configuring AWS-specific options
|
||||||
|
ywang29-p4d-debug-1-worker-0:212865:212865 [4] NCCL INFO NET/OFI Setting provider_filter to efa
|
||||||
|
ywang29-p4d-debug-1-worker-0:212865:212865 [4] NCCL INFO NET/OFI Running on p4de.24xlarge platform, topology file /opt/amazon/ofi-nccl/share/aws-ofi-nccl/xml/p4de-24xl-topo.xml
|
||||||
|
ywang29-p4d-debug-1-worker-0:212865:212865 [4] NCCL INFO NET/OFI Internode latency set at 75.0 us
|
||||||
|
ywang29-p4d-debug-1-worker-0:212865:212865 [4] NCCL INFO NET/OFI Using transport protocol SENDRECV (platform set)
|
||||||
|
|
||||||
|
[2026-04-29 15:37:18] ywang29-p4d-debug-1-worker-0:212865:212865 [4] int nccl_net_ofi_create_plugin(nccl_net_ofi_plugin_t**):208 NCCL WARN NET/OFI Failed to initialize sendrecv protocol
|
||||||
|
|
||||||
|
[2026-04-29 15:37:18] ywang29-p4d-debug-1-worker-0:212865:212865 [4] int nccl_net_ofi_create_plugin(nccl_net_ofi_plugin_t**):353 NCCL WARN NET/OFI aws-ofi-nccl initialization failed
|
||||||
|
|
||||||
|
[2026-04-29 15:37:18] ywang29-p4d-debug-1-worker-0:212865:212865 [4] ncclResult_t nccl_net_ofi_init_v6(ncclDebugLogger_t):162 NCCL WARN NET/OFI Initializing plugin failed
|
||||||
|
ywang29-p4d-debug-1-worker-0:212865:212865 [4] NCCL INFO plugin/net/net_v10.cc:81 -> 2
|
||||||
|
ywang29-p4d-debug-1-worker-0:212865:212865 [4] NCCL INFO Failed to initialize NET plugin Libfabric
|
||||||
|
ywang29-p4d-debug-1-worker-0:212868:212868 [7] NCCL INFO NET/Plugin: Loaded net plugin Libfabric (v10)
|
||||||
|
ywang29-p4d-debug-1-worker-0:212868:212868 [7] NCCL INFO Successfully loaded external network plugin /opt/amazon/ofi-nccl/lib/libnccl-net.so
|
||||||
|
ywang29-p4d-debug-1-worker-0:212868:212868 [7] NCCL INFO NET/OFI Initializing aws-ofi-nccl 1.17.1
|
||||||
|
ywang29-p4d-debug-1-worker-0:212868:212868 [7] NCCL INFO NET/OFI Using Libfabric version 2.3
|
||||||
|
ywang29-p4d-debug-1-worker-0:212868:212868 [7] NCCL INFO NET/OFI Using CUDA driver version 13000 with runtime 13000
|
||||||
|
ywang29-p4d-debug-1-worker-0:212868:212868 [7] NCCL INFO NET/OFI Configuring AWS-specific options
|
||||||
|
ywang29-p4d-debug-1-worker-0:212868:212868 [7] NCCL INFO NET/OFI Setting provider_filter to efa
|
||||||
|
ywang29-p4d-debug-1-worker-0:212868:212868 [7] NCCL INFO NET/OFI Running on p4de.24xlarge platform, topology file /opt/amazon/ofi-nccl/share/aws-ofi-nccl/xml/p4de-24xl-topo.xml
|
||||||
|
ywang29-p4d-debug-1-worker-0:212868:212868 [7] NCCL INFO NET/OFI Internode latency set at 75.0 us
|
||||||
|
ywang29-p4d-debug-1-worker-0:212868:212868 [7] NCCL INFO NET/OFI Using transport protocol SENDRECV (platform set)
|
||||||
|
|
||||||
|
[2026-04-29 15:37:18] ywang29-p4d-debug-1-worker-0:212868:212868 [7] int nccl_net_ofi_create_plugin(nccl_net_ofi_plugin_t**):208 NCCL WARN NET/OFI Failed to initialize sendrecv protocol
|
||||||
|
|
||||||
|
[2026-04-29 15:37:18] ywang29-p4d-debug-1-worker-0:212868:212868 [7] int nccl_net_ofi_create_plugin(nccl_net_ofi_plugin_t**):353 NCCL WARN NET/OFI aws-ofi-nccl initialization failed
|
||||||
|
|
||||||
|
[2026-04-29 15:37:18] ywang29-p4d-debug-1-worker-0:212868:212868 [7] ncclResult_t nccl_net_ofi_init_v6(ncclDebugLogger_t):162 NCCL WARN NET/OFI Initializing plugin failed
|
||||||
|
ywang29-p4d-debug-1-worker-0:212868:212868 [7] NCCL INFO plugin/net/net_v10.cc:81 -> 2
|
||||||
|
ywang29-p4d-debug-1-worker-0:212868:212868 [7] NCCL INFO Failed to initialize NET plugin Libfabric
|
||||||
|
ywang29-p4d-debug-1-worker-0:212861:212861 [0] NCCL INFO NET/Plugin: Loaded net plugin Libfabric (v10)
|
||||||
|
ywang29-p4d-debug-1-worker-0:212861:212861 [0] NCCL INFO Successfully loaded external network plugin /opt/amazon/ofi-nccl/lib/libnccl-net.so
|
||||||
|
ywang29-p4d-debug-1-worker-0:212861:212861 [0] NCCL INFO NET/OFI Initializing aws-ofi-nccl 1.17.1
|
||||||
|
ywang29-p4d-debug-1-worker-0:212861:212861 [0] NCCL INFO NET/OFI Using Libfabric version 2.3
|
||||||
|
ywang29-p4d-debug-1-worker-0:212861:212861 [0] NCCL INFO NET/OFI Using CUDA driver version 13000 with runtime 13000
|
||||||
|
ywang29-p4d-debug-1-worker-0:212861:212861 [0] NCCL INFO NET/OFI Configuring AWS-specific options
|
||||||
|
ywang29-p4d-debug-1-worker-0:212861:212861 [0] NCCL INFO NET/OFI Setting provider_filter to efa
|
||||||
|
ywang29-p4d-debug-1-worker-0:212861:212861 [0] NCCL INFO NET/OFI Running on p4de.24xlarge platform, topology file /opt/amazon/ofi-nccl/share/aws-ofi-nccl/xml/p4de-24xl-topo.xml
|
||||||
|
ywang29-p4d-debug-1-worker-0:212861:212861 [0] NCCL INFO NET/OFI Internode latency set at 75.0 us
|
||||||
|
ywang29-p4d-debug-1-worker-0:212861:212861 [0] NCCL INFO NET/OFI Using transport protocol SENDRECV (platform set)
|
||||||
|
|
||||||
|
[2026-04-29 15:37:19] ywang29-p4d-debug-1-worker-0:212861:212861 [0] int nccl_net_ofi_create_plugin(nccl_net_ofi_plugin_t**):208 NCCL WARN NET/OFI Failed to initialize sendrecv protocol
|
||||||
|
|
||||||
|
[2026-04-29 15:37:19] ywang29-p4d-debug-1-worker-0:212861:212861 [0] int nccl_net_ofi_create_plugin(nccl_net_ofi_plugin_t**):353 NCCL WARN NET/OFI aws-ofi-nccl initialization failed
|
||||||
|
|
||||||
|
[2026-04-29 15:37:19] ywang29-p4d-debug-1-worker-0:212861:212861 [0] ncclResult_t nccl_net_ofi_init_v6(ncclDebugLogger_t):162 NCCL WARN NET/OFI Initializing plugin failed
|
||||||
|
ywang29-p4d-debug-1-worker-0:212861:212861 [0] NCCL INFO plugin/net/net_v10.cc:81 -> 2
|
||||||
|
ywang29-p4d-debug-1-worker-0:212861:212861 [0] NCCL INFO Failed to initialize NET plugin Libfabric
|
||||||
|
ywang29-p4d-debug-1-worker-0:212864:212864 [3] NCCL INFO NET/Plugin: Loaded net plugin Libfabric (v10)
|
||||||
|
ywang29-p4d-debug-1-worker-0:212864:212864 [3] NCCL INFO Successfully loaded external network plugin /opt/amazon/ofi-nccl/lib/libnccl-net.so
|
||||||
|
ywang29-p4d-debug-1-worker-0:212864:212864 [3] NCCL INFO NET/OFI Initializing aws-ofi-nccl 1.17.1
|
||||||
|
ywang29-p4d-debug-1-worker-0:212864:212864 [3] NCCL INFO NET/OFI Using Libfabric version 2.3
|
||||||
|
ywang29-p4d-debug-1-worker-0:212864:212864 [3] NCCL INFO NET/OFI Using CUDA driver version 13000 with runtime 13000
|
||||||
|
ywang29-p4d-debug-1-worker-0:212864:212864 [3] NCCL INFO NET/OFI Configuring AWS-specific options
|
||||||
|
ywang29-p4d-debug-1-worker-0:212864:212864 [3] NCCL INFO NET/OFI Setting provider_filter to efa
|
||||||
|
ywang29-p4d-debug-1-worker-0:212864:212864 [3] NCCL INFO NET/OFI Running on p4de.24xlarge platform, topology file /opt/amazon/ofi-nccl/share/aws-ofi-nccl/xml/p4de-24xl-topo.xml
|
||||||
|
ywang29-p4d-debug-1-worker-0:212864:212864 [3] NCCL INFO NET/OFI Internode latency set at 75.0 us
|
||||||
|
ywang29-p4d-debug-1-worker-0:212864:212864 [3] NCCL INFO NET/OFI Using transport protocol SENDRECV (platform set)
|
||||||
|
|
||||||
|
[2026-04-29 15:37:19] ywang29-p4d-debug-1-worker-0:212864:212864 [3] int nccl_net_ofi_create_plugin(nccl_net_ofi_plugin_t**):208 NCCL WARN NET/OFI Failed to initialize sendrecv protocol
|
||||||
|
|
||||||
|
[2026-04-29 15:37:19] ywang29-p4d-debug-1-worker-0:212864:212864 [3] int nccl_net_ofi_create_plugin(nccl_net_ofi_plugin_t**):353 NCCL WARN NET/OFI aws-ofi-nccl initialization failed
|
||||||
|
|
||||||
|
[2026-04-29 15:37:19] ywang29-p4d-debug-1-worker-0:212864:212864 [3] ncclResult_t nccl_net_ofi_init_v6(ncclDebugLogger_t):162 NCCL WARN NET/OFI Initializing plugin failed
|
||||||
|
ywang29-p4d-debug-1-worker-0:212864:212864 [3] NCCL INFO plugin/net/net_v10.cc:81 -> 2
|
||||||
|
ywang29-p4d-debug-1-worker-0:212864:212864 [3] NCCL INFO Failed to initialize NET plugin Libfabric
|
||||||
|
ywang29-p4d-debug-1-worker-0:212866:212866 [5] NCCL INFO NET/Plugin: Loaded net plugin Libfabric (v10)
|
||||||
|
ywang29-p4d-debug-1-worker-0:212866:212866 [5] NCCL INFO Successfully loaded external network plugin /opt/amazon/ofi-nccl/lib/libnccl-net.so
|
||||||
|
ywang29-p4d-debug-1-worker-0:212866:212866 [5] NCCL INFO NET/OFI Initializing aws-ofi-nccl 1.17.1
|
||||||
|
ywang29-p4d-debug-1-worker-0:212866:212866 [5] NCCL INFO NET/OFI Using Libfabric version 2.3
|
||||||
|
ywang29-p4d-debug-1-worker-0:212866:212866 [5] NCCL INFO NET/OFI Using CUDA driver version 13000 with runtime 13000
|
||||||
|
ywang29-p4d-debug-1-worker-0:212866:212866 [5] NCCL INFO NET/OFI Configuring AWS-specific options
|
||||||
|
ywang29-p4d-debug-1-worker-0:212866:212866 [5] NCCL INFO NET/OFI Setting provider_filter to efa
|
||||||
|
ywang29-p4d-debug-1-worker-0:212866:212866 [5] NCCL INFO NET/OFI Running on p4de.24xlarge platform, topology file /opt/amazon/ofi-nccl/share/aws-ofi-nccl/xml/p4de-24xl-topo.xml
|
||||||
|
ywang29-p4d-debug-1-worker-0:212866:212866 [5] NCCL INFO NET/OFI Internode latency set at 75.0 us
|
||||||
|
ywang29-p4d-debug-1-worker-0:212866:212866 [5] NCCL INFO NET/OFI Using transport protocol SENDRECV (platform set)
|
||||||
|
|
||||||
|
[2026-04-29 15:37:19] ywang29-p4d-debug-1-worker-0:212866:212866 [5] int nccl_net_ofi_create_plugin(nccl_net_ofi_plugin_t**):208 NCCL WARN NET/OFI Failed to initialize sendrecv protocol
|
||||||
|
|
||||||
|
[2026-04-29 15:37:19] ywang29-p4d-debug-1-worker-0:212866:212866 [5] int nccl_net_ofi_create_plugin(nccl_net_ofi_plugin_t**):353 NCCL WARN NET/OFI aws-ofi-nccl initialization failed
|
||||||
|
|
||||||
|
[2026-04-29 15:37:19] ywang29-p4d-debug-1-worker-0:212866:212866 [5] ncclResult_t nccl_net_ofi_init_v6(ncclDebugLogger_t):162 NCCL WARN NET/OFI Initializing plugin failed
|
||||||
|
ywang29-p4d-debug-1-worker-0:212866:212866 [5] NCCL INFO plugin/net/net_v10.cc:81 -> 2
|
||||||
|
ywang29-p4d-debug-1-worker-0:212866:212866 [5] NCCL INFO Failed to initialize NET plugin Libfabric
|
||||||
1343
qwen3_1.7b_coding_fft_20260429_153746.log
Normal file
1343
qwen3_1.7b_coding_fft_20260429_153746.log
Normal file
File diff suppressed because one or more lines are too long
2823
qwen3_1.7b_coding_fft_20260429_154000.log
Normal file
2823
qwen3_1.7b_coding_fft_20260429_154000.log
Normal file
File diff suppressed because one or more lines are too long
2844
qwen3_1.7b_coding_fft_20260429_170521.log
Normal file
2844
qwen3_1.7b_coding_fft_20260429_170521.log
Normal file
File diff suppressed because one or more lines are too long
3
tokenizer.json
Normal file
3
tokenizer.json
Normal file
@@ -0,0 +1,3 @@
|
|||||||
|
version https://git-lfs.github.com/spec/v1
|
||||||
|
oid sha256:be75606093db2094d7cd20f3c2f385c212750648bd6ea4fb2bf507a6a4c55506
|
||||||
|
size 11422650
|
||||||
30
tokenizer_config.json
Normal file
30
tokenizer_config.json
Normal file
@@ -0,0 +1,30 @@
|
|||||||
|
{
|
||||||
|
"add_prefix_space": false,
|
||||||
|
"backend": "tokenizers",
|
||||||
|
"bos_token": null,
|
||||||
|
"clean_up_tokenization_spaces": false,
|
||||||
|
"eos_token": "<|im_end|>",
|
||||||
|
"errors": "replace",
|
||||||
|
"extra_special_tokens": [
|
||||||
|
"<|im_start|>",
|
||||||
|
"<|im_end|>",
|
||||||
|
"<|object_ref_start|>",
|
||||||
|
"<|object_ref_end|>",
|
||||||
|
"<|box_start|>",
|
||||||
|
"<|box_end|>",
|
||||||
|
"<|quad_start|>",
|
||||||
|
"<|quad_end|>",
|
||||||
|
"<|vision_start|>",
|
||||||
|
"<|vision_end|>",
|
||||||
|
"<|vision_pad|>",
|
||||||
|
"<|image_pad|>",
|
||||||
|
"<|video_pad|>"
|
||||||
|
],
|
||||||
|
"is_local": false,
|
||||||
|
"model_max_length": 131072,
|
||||||
|
"pad_token": "<|endoftext|>",
|
||||||
|
"padding_side": "right",
|
||||||
|
"split_special_tokens": false,
|
||||||
|
"tokenizer_class": "Qwen2Tokenizer",
|
||||||
|
"unk_token": null
|
||||||
|
}
|
||||||
8
train_results.json
Normal file
8
train_results.json
Normal file
@@ -0,0 +1,8 @@
|
|||||||
|
{
|
||||||
|
"epoch": 4.0,
|
||||||
|
"total_flos": 7.974893111623025e+17,
|
||||||
|
"train_loss": 0.6604051219656112,
|
||||||
|
"train_runtime": 3155.399,
|
||||||
|
"train_samples_per_second": 38.03,
|
||||||
|
"train_steps_per_second": 0.298
|
||||||
|
}
|
||||||
95
trainer_log.jsonl
Normal file
95
trainer_log.jsonl
Normal file
@@ -0,0 +1,95 @@
|
|||||||
|
{"current_steps": 10, "total_steps": 940, "loss": 1.291497802734375, "lr": 9.574468085106384e-07, "epoch": 0.042666666666666665, "percentage": 1.06, "elapsed_time": "0:00:35", "remaining_time": "0:55:03"}
|
||||||
|
{"current_steps": 20, "total_steps": 940, "loss": 1.182865810394287, "lr": 2.021276595744681e-06, "epoch": 0.08533333333333333, "percentage": 2.13, "elapsed_time": "0:01:10", "remaining_time": "0:53:55"}
|
||||||
|
{"current_steps": 30, "total_steps": 940, "loss": 0.9575789451599122, "lr": 3.0851063829787237e-06, "epoch": 0.128, "percentage": 3.19, "elapsed_time": "0:01:43", "remaining_time": "0:52:21"}
|
||||||
|
{"current_steps": 40, "total_steps": 940, "loss": 0.8388131141662598, "lr": 4.148936170212766e-06, "epoch": 0.17066666666666666, "percentage": 4.26, "elapsed_time": "0:02:13", "remaining_time": "0:49:54"}
|
||||||
|
{"current_steps": 50, "total_steps": 940, "loss": 0.8148186683654786, "lr": 5.212765957446809e-06, "epoch": 0.21333333333333335, "percentage": 5.32, "elapsed_time": "0:02:50", "remaining_time": "0:50:42"}
|
||||||
|
{"current_steps": 60, "total_steps": 940, "loss": 0.7651220321655273, "lr": 6.276595744680851e-06, "epoch": 0.256, "percentage": 6.38, "elapsed_time": "0:03:22", "remaining_time": "0:49:25"}
|
||||||
|
{"current_steps": 70, "total_steps": 940, "loss": 0.7899383544921875, "lr": 7.340425531914894e-06, "epoch": 0.2986666666666667, "percentage": 7.45, "elapsed_time": "0:03:57", "remaining_time": "0:49:15"}
|
||||||
|
{"current_steps": 80, "total_steps": 940, "loss": 0.7782301902770996, "lr": 8.404255319148937e-06, "epoch": 0.3413333333333333, "percentage": 8.51, "elapsed_time": "0:04:30", "remaining_time": "0:48:25"}
|
||||||
|
{"current_steps": 90, "total_steps": 940, "loss": 0.7684392929077148, "lr": 9.46808510638298e-06, "epoch": 0.384, "percentage": 9.57, "elapsed_time": "0:05:07", "remaining_time": "0:48:22"}
|
||||||
|
{"current_steps": 100, "total_steps": 940, "loss": 0.7520565032958985, "lr": 9.999138160172624e-06, "epoch": 0.4266666666666667, "percentage": 10.64, "elapsed_time": "0:05:40", "remaining_time": "0:47:40"}
|
||||||
|
{"current_steps": 110, "total_steps": 940, "loss": 0.7115557670593262, "lr": 9.99224522409411e-06, "epoch": 0.4693333333333333, "percentage": 11.7, "elapsed_time": "0:06:18", "remaining_time": "0:47:35"}
|
||||||
|
{"current_steps": 120, "total_steps": 940, "loss": 0.7434285163879395, "lr": 9.97846885608885e-06, "epoch": 0.512, "percentage": 12.77, "elapsed_time": "0:06:51", "remaining_time": "0:46:54"}
|
||||||
|
{"current_steps": 130, "total_steps": 940, "loss": 0.7457537651062012, "lr": 9.957828051355817e-06, "epoch": 0.5546666666666666, "percentage": 13.83, "elapsed_time": "0:07:29", "remaining_time": "0:46:41"}
|
||||||
|
{"current_steps": 140, "total_steps": 940, "loss": 0.7388599872589111, "lr": 9.930351269950144e-06, "epoch": 0.5973333333333334, "percentage": 14.89, "elapsed_time": "0:08:05", "remaining_time": "0:46:11"}
|
||||||
|
{"current_steps": 150, "total_steps": 940, "loss": 0.7285007476806641, "lr": 9.896076397541676e-06, "epoch": 0.64, "percentage": 15.96, "elapsed_time": "0:08:36", "remaining_time": "0:45:21"}
|
||||||
|
{"current_steps": 160, "total_steps": 940, "loss": 0.7659842491149902, "lr": 9.855050693177286e-06, "epoch": 0.6826666666666666, "percentage": 17.02, "elapsed_time": "0:09:08", "remaining_time": "0:44:33"}
|
||||||
|
{"current_steps": 170, "total_steps": 940, "loss": 0.724073839187622, "lr": 9.807330724118906e-06, "epoch": 0.7253333333333334, "percentage": 18.09, "elapsed_time": "0:09:45", "remaining_time": "0:44:11"}
|
||||||
|
{"current_steps": 180, "total_steps": 940, "loss": 0.747947883605957, "lr": 9.752982287847193e-06, "epoch": 0.768, "percentage": 19.15, "elapsed_time": "0:10:17", "remaining_time": "0:43:26"}
|
||||||
|
{"current_steps": 190, "total_steps": 940, "loss": 0.7283991813659668, "lr": 9.692080321338317e-06, "epoch": 0.8106666666666666, "percentage": 20.21, "elapsed_time": "0:10:48", "remaining_time": "0:42:41"}
|
||||||
|
{"current_steps": 200, "total_steps": 940, "loss": 0.7407590389251709, "lr": 9.624708797739002e-06, "epoch": 0.8533333333333334, "percentage": 21.28, "elapsed_time": "0:11:16", "remaining_time": "0:41:43"}
|
||||||
|
{"current_steps": 210, "total_steps": 940, "loss": 0.724196481704712, "lr": 9.550960610582251e-06, "epoch": 0.896, "percentage": 22.34, "elapsed_time": "0:11:50", "remaining_time": "0:41:11"}
|
||||||
|
{"current_steps": 220, "total_steps": 940, "loss": 0.7334166526794433, "lr": 9.47093744570344e-06, "epoch": 0.9386666666666666, "percentage": 23.4, "elapsed_time": "0:12:20", "remaining_time": "0:40:22"}
|
||||||
|
{"current_steps": 230, "total_steps": 940, "loss": 0.8169802665710449, "lr": 9.384749641033358e-06, "epoch": 0.9813333333333333, "percentage": 24.47, "elapsed_time": "0:12:49", "remaining_time": "0:39:34"}
|
||||||
|
{"current_steps": 240, "total_steps": 940, "loss": 0.7211981773376465, "lr": 9.292516034461517e-06, "epoch": 1.0213333333333334, "percentage": 25.53, "elapsed_time": "0:13:20", "remaining_time": "0:38:54"}
|
||||||
|
{"current_steps": 250, "total_steps": 940, "loss": 0.6335953712463379, "lr": 9.194363799979517e-06, "epoch": 1.064, "percentage": 26.6, "elapsed_time": "0:13:52", "remaining_time": "0:38:17"}
|
||||||
|
{"current_steps": 260, "total_steps": 940, "loss": 0.6689927577972412, "lr": 9.090428272330381e-06, "epoch": 1.1066666666666667, "percentage": 27.66, "elapsed_time": "0:14:23", "remaining_time": "0:37:38"}
|
||||||
|
{"current_steps": 270, "total_steps": 940, "loss": 0.6809295654296875, "lr": 8.980852760405645e-06, "epoch": 1.1493333333333333, "percentage": 28.72, "elapsed_time": "0:14:59", "remaining_time": "0:37:10"}
|
||||||
|
{"current_steps": 280, "total_steps": 940, "loss": 0.6914172172546387, "lr": 8.865788349647496e-06, "epoch": 1.192, "percentage": 29.79, "elapsed_time": "0:15:27", "remaining_time": "0:36:25"}
|
||||||
|
{"current_steps": 290, "total_steps": 940, "loss": 0.6650928974151611, "lr": 8.745393693728395e-06, "epoch": 1.2346666666666666, "percentage": 30.85, "elapsed_time": "0:16:00", "remaining_time": "0:35:53"}
|
||||||
|
{"current_steps": 300, "total_steps": 940, "loss": 0.6717124938964844, "lr": 8.619834795795458e-06, "epoch": 1.2773333333333334, "percentage": 31.91, "elapsed_time": "0:16:30", "remaining_time": "0:35:12"}
|
||||||
|
{"current_steps": 310, "total_steps": 940, "loss": 0.6726479530334473, "lr": 8.489284779581179e-06, "epoch": 1.32, "percentage": 32.98, "elapsed_time": "0:17:06", "remaining_time": "0:34:46"}
|
||||||
|
{"current_steps": 320, "total_steps": 940, "loss": 0.6797503471374512, "lr": 8.353923650696119e-06, "epoch": 1.3626666666666667, "percentage": 34.04, "elapsed_time": "0:17:37", "remaining_time": "0:34:08"}
|
||||||
|
{"current_steps": 330, "total_steps": 940, "loss": 0.6944191932678223, "lr": 8.213938048432697e-06, "epoch": 1.4053333333333333, "percentage": 35.11, "elapsed_time": "0:18:12", "remaining_time": "0:33:39"}
|
||||||
|
{"current_steps": 340, "total_steps": 940, "loss": 0.619914960861206, "lr": 8.069520988422292e-06, "epoch": 1.448, "percentage": 36.17, "elapsed_time": "0:18:46", "remaining_time": "0:33:07"}
|
||||||
|
{"current_steps": 350, "total_steps": 940, "loss": 0.6835647583007812, "lr": 7.920871596500473e-06, "epoch": 1.4906666666666666, "percentage": 37.23, "elapsed_time": "0:19:17", "remaining_time": "0:32:30"}
|
||||||
|
{"current_steps": 360, "total_steps": 940, "loss": 0.6706738471984863, "lr": 7.768194834147362e-06, "epoch": 1.5333333333333332, "percentage": 38.3, "elapsed_time": "0:19:49", "remaining_time": "0:31:55"}
|
||||||
|
{"current_steps": 370, "total_steps": 940, "loss": 0.690638542175293, "lr": 7.611701215881635e-06, "epoch": 1.576, "percentage": 39.36, "elapsed_time": "0:20:27", "remaining_time": "0:31:31"}
|
||||||
|
{"current_steps": 380, "total_steps": 940, "loss": 0.6714137077331543, "lr": 7.4516065189978625e-06, "epoch": 1.6186666666666667, "percentage": 40.43, "elapsed_time": "0:21:01", "remaining_time": "0:30:59"}
|
||||||
|
{"current_steps": 390, "total_steps": 940, "loss": 0.6779637336730957, "lr": 7.288131486047414e-06, "epoch": 1.6613333333333333, "percentage": 41.49, "elapsed_time": "0:21:36", "remaining_time": "0:30:28"}
|
||||||
|
{"current_steps": 400, "total_steps": 940, "loss": 0.6494068622589111, "lr": 7.121501520473137e-06, "epoch": 1.704, "percentage": 42.55, "elapsed_time": "0:22:13", "remaining_time": "0:30:00"}
|
||||||
|
{"current_steps": 410, "total_steps": 940, "loss": 0.6467916011810303, "lr": 6.9519463758174745e-06, "epoch": 1.7466666666666666, "percentage": 43.62, "elapsed_time": "0:22:48", "remaining_time": "0:29:29"}
|
||||||
|
{"current_steps": 420, "total_steps": 940, "loss": 0.6865511894226074, "lr": 6.77969983893257e-06, "epoch": 1.7893333333333334, "percentage": 44.68, "elapsed_time": "0:23:22", "remaining_time": "0:28:56"}
|
||||||
|
{"current_steps": 430, "total_steps": 940, "loss": 0.6830032348632813, "lr": 6.604999407629137e-06, "epoch": 1.8319999999999999, "percentage": 45.74, "elapsed_time": "0:23:57", "remaining_time": "0:28:25"}
|
||||||
|
{"current_steps": 440, "total_steps": 940, "loss": 0.6937445640563965, "lr": 6.428085963208567e-06, "epoch": 1.8746666666666667, "percentage": 46.81, "elapsed_time": "0:24:32", "remaining_time": "0:27:53"}
|
||||||
|
{"current_steps": 450, "total_steps": 940, "loss": 0.6877000331878662, "lr": 6.249203438329799e-06, "epoch": 1.9173333333333333, "percentage": 47.87, "elapsed_time": "0:25:08", "remaining_time": "0:27:22"}
|
||||||
|
{"current_steps": 460, "total_steps": 940, "loss": 0.6529797077178955, "lr": 6.0685984806689055e-06, "epoch": 1.96, "percentage": 48.94, "elapsed_time": "0:25:43", "remaining_time": "0:26:50"}
|
||||||
|
{"current_steps": 470, "total_steps": 940, "loss": 0.6860092163085938, "lr": 5.886520112835128e-06, "epoch": 2.0, "percentage": 50.0, "elapsed_time": "0:26:15", "remaining_time": "0:26:15"}
|
||||||
|
{"current_steps": 480, "total_steps": 940, "loss": 0.6075316905975342, "lr": 5.703219389012317e-06, "epoch": 2.042666666666667, "percentage": 51.06, "elapsed_time": "0:26:48", "remaining_time": "0:25:41"}
|
||||||
|
{"current_steps": 490, "total_steps": 940, "loss": 0.5943183422088623, "lr": 5.518949048799176e-06, "epoch": 2.0853333333333333, "percentage": 52.13, "elapsed_time": "0:27:20", "remaining_time": "0:25:07"}
|
||||||
|
{"current_steps": 500, "total_steps": 940, "loss": 0.5897200107574463, "lr": 5.3339631687256085e-06, "epoch": 2.128, "percentage": 53.19, "elapsed_time": "0:27:53", "remaining_time": "0:24:32"}
|
||||||
|
{"current_steps": 510, "total_steps": 940, "loss": 0.6106101036071777, "lr": 5.148516811925684e-06, "epoch": 2.1706666666666665, "percentage": 54.26, "elapsed_time": "0:28:36", "remaining_time": "0:24:07"}
|
||||||
|
{"current_steps": 520, "total_steps": 940, "loss": 0.6104888916015625, "lr": 4.962865676450239e-06, "epoch": 2.2133333333333334, "percentage": 55.32, "elapsed_time": "0:29:09", "remaining_time": "0:23:32"}
|
||||||
|
{"current_steps": 530, "total_steps": 940, "loss": 0.5798359870910644, "lr": 4.777265742704039e-06, "epoch": 2.2560000000000002, "percentage": 56.38, "elapsed_time": "0:29:42", "remaining_time": "0:22:59"}
|
||||||
|
{"current_steps": 540, "total_steps": 940, "loss": 0.6197110652923584, "lr": 4.591972920493638e-06, "epoch": 2.2986666666666666, "percentage": 57.45, "elapsed_time": "0:30:17", "remaining_time": "0:22:26"}
|
||||||
|
{"current_steps": 550, "total_steps": 940, "loss": 0.5831963539123535, "lr": 4.40724269617256e-06, "epoch": 2.3413333333333335, "percentage": 58.51, "elapsed_time": "0:30:50", "remaining_time": "0:21:52"}
|
||||||
|
{"current_steps": 560, "total_steps": 940, "loss": 0.598298454284668, "lr": 4.223329780370359e-06, "epoch": 2.384, "percentage": 59.57, "elapsed_time": "0:31:24", "remaining_time": "0:21:18"}
|
||||||
|
{"current_steps": 570, "total_steps": 940, "loss": 0.5884403705596923, "lr": 4.04048775679127e-06, "epoch": 2.4266666666666667, "percentage": 60.64, "elapsed_time": "0:32:00", "remaining_time": "0:20:46"}
|
||||||
|
{"current_steps": 580, "total_steps": 940, "loss": 0.5839417457580567, "lr": 3.858968732566685e-06, "epoch": 2.469333333333333, "percentage": 61.7, "elapsed_time": "0:32:34", "remaining_time": "0:20:13"}
|
||||||
|
{"current_steps": 590, "total_steps": 940, "loss": 0.6246652126312255, "lr": 3.6790229906435706e-06, "epoch": 2.512, "percentage": 62.77, "elapsed_time": "0:33:07", "remaining_time": "0:19:38"}
|
||||||
|
{"current_steps": 600, "total_steps": 940, "loss": 0.6042469024658204, "lr": 3.5008986446881088e-06, "epoch": 2.554666666666667, "percentage": 63.83, "elapsed_time": "0:33:48", "remaining_time": "0:19:09"}
|
||||||
|
{"current_steps": 610, "total_steps": 940, "loss": 0.6285554885864257, "lr": 3.3248412969804065e-06, "epoch": 2.5973333333333333, "percentage": 64.89, "elapsed_time": "0:34:21", "remaining_time": "0:18:35"}
|
||||||
|
{"current_steps": 620, "total_steps": 940, "loss": 0.6006599426269531, "lr": 3.1510936997719557e-06, "epoch": 2.64, "percentage": 65.96, "elapsed_time": "0:34:55", "remaining_time": "0:18:01"}
|
||||||
|
{"current_steps": 630, "total_steps": 940, "loss": 0.5928638458251954, "lr": 2.9798954205727886e-06, "epoch": 2.6826666666666665, "percentage": 67.02, "elapsed_time": "0:35:25", "remaining_time": "0:17:25"}
|
||||||
|
{"current_steps": 640, "total_steps": 940, "loss": 0.6103617668151855, "lr": 2.811482511829842e-06, "epoch": 2.7253333333333334, "percentage": 68.09, "elapsed_time": "0:35:58", "remaining_time": "0:16:51"}
|
||||||
|
{"current_steps": 650, "total_steps": 940, "loss": 0.5874536514282227, "lr": 2.6460871854519594e-06, "epoch": 2.768, "percentage": 69.15, "elapsed_time": "0:36:31", "remaining_time": "0:16:17"}
|
||||||
|
{"current_steps": 660, "total_steps": 940, "loss": 0.5812715053558349, "lr": 2.483937492630345e-06, "epoch": 2.8106666666666666, "percentage": 70.21, "elapsed_time": "0:37:04", "remaining_time": "0:15:43"}
|
||||||
|
{"current_steps": 670, "total_steps": 940, "loss": 0.609982681274414, "lr": 2.3252570093959e-06, "epoch": 2.8533333333333335, "percentage": 71.28, "elapsed_time": "0:37:37", "remaining_time": "0:15:09"}
|
||||||
|
{"current_steps": 680, "total_steps": 940, "loss": 0.6121101379394531, "lr": 2.1702645283470238e-06, "epoch": 2.896, "percentage": 72.34, "elapsed_time": "0:38:08", "remaining_time": "0:14:34"}
|
||||||
|
{"current_steps": 690, "total_steps": 940, "loss": 0.6171843528747558, "lr": 2.0191737569729492e-06, "epoch": 2.9386666666666668, "percentage": 73.4, "elapsed_time": "0:38:41", "remaining_time": "0:14:01"}
|
||||||
|
{"current_steps": 700, "total_steps": 940, "loss": 0.5880200386047363, "lr": 1.872193022988526e-06, "epoch": 2.981333333333333, "percentage": 74.47, "elapsed_time": "0:39:14", "remaining_time": "0:13:27"}
|
||||||
|
{"current_steps": 710, "total_steps": 940, "loss": 0.5795096397399903, "lr": 1.7295249870867898e-06, "epoch": 3.021333333333333, "percentage": 75.53, "elapsed_time": "0:39:46", "remaining_time": "0:12:53"}
|
||||||
|
{"current_steps": 720, "total_steps": 940, "loss": 0.5702518463134766, "lr": 1.5913663635053578e-06, "epoch": 3.064, "percentage": 76.6, "elapsed_time": "0:40:18", "remaining_time": "0:12:19"}
|
||||||
|
{"current_steps": 730, "total_steps": 940, "loss": 0.5839662551879883, "lr": 1.457907648791943e-06, "epoch": 3.1066666666666665, "percentage": 77.66, "elapsed_time": "0:40:51", "remaining_time": "0:11:45"}
|
||||||
|
{"current_steps": 740, "total_steps": 940, "loss": 0.5281820297241211, "lr": 1.329332859142967e-06, "epoch": 3.1493333333333333, "percentage": 78.72, "elapsed_time": "0:41:24", "remaining_time": "0:11:11"}
|
||||||
|
{"current_steps": 750, "total_steps": 940, "loss": 0.5751385688781738, "lr": 1.205819276677464e-06, "epoch": 3.192, "percentage": 79.79, "elapsed_time": "0:41:59", "remaining_time": "0:10:38"}
|
||||||
|
{"current_steps": 760, "total_steps": 940, "loss": 0.5494537353515625, "lr": 1.0875372049960697e-06, "epoch": 3.2346666666666666, "percentage": 80.85, "elapsed_time": "0:42:31", "remaining_time": "0:10:04"}
|
||||||
|
{"current_steps": 770, "total_steps": 940, "loss": 0.5537368297576905, "lr": 9.746497343621857e-07, "epoch": 3.2773333333333334, "percentage": 81.91, "elapsed_time": "0:43:06", "remaining_time": "0:09:31"}
|
||||||
|
{"current_steps": 780, "total_steps": 940, "loss": 0.55874662399292, "lr": 8.673125168290713e-07, "epoch": 3.32, "percentage": 82.98, "elapsed_time": "0:43:40", "remaining_time": "0:08:57"}
|
||||||
|
{"current_steps": 790, "total_steps": 940, "loss": 0.565284252166748, "lr": 7.656735516229125e-07, "epoch": 3.3626666666666667, "percentage": 84.04, "elapsed_time": "0:44:11", "remaining_time": "0:08:23"}
|
||||||
|
{"current_steps": 800, "total_steps": 940, "loss": 0.5531235694885254, "lr": 6.698729810778065e-07, "epoch": 3.405333333333333, "percentage": 85.11, "elapsed_time": "0:44:48", "remaining_time": "0:07:50"}
|
||||||
|
{"current_steps": 810, "total_steps": 940, "loss": 0.5778326511383056, "lr": 5.800428974040311e-07, "epoch": 3.448, "percentage": 86.17, "elapsed_time": "0:45:24", "remaining_time": "0:07:17"}
|
||||||
|
{"current_steps": 820, "total_steps": 940, "loss": 0.5518184661865234, "lr": 4.963071605560144e-07, "epoch": 3.490666666666667, "percentage": 87.23, "elapsed_time": "0:46:01", "remaining_time": "0:06:44"}
|
||||||
|
{"current_steps": 830, "total_steps": 940, "loss": 0.5773732185363769, "lr": 4.187812274511427e-07, "epoch": 3.533333333333333, "percentage": 88.3, "elapsed_time": "0:46:31", "remaining_time": "0:06:09"}
|
||||||
|
{"current_steps": 840, "total_steps": 940, "loss": 0.5663552284240723, "lr": 3.4757199277490106e-07, "epoch": 3.576, "percentage": 89.36, "elapsed_time": "0:47:03", "remaining_time": "0:05:36"}
|
||||||
|
{"current_steps": 850, "total_steps": 940, "loss": 0.5664983272552491, "lr": 2.8277764159181484e-07, "epoch": 3.618666666666667, "percentage": 90.43, "elapsed_time": "0:47:36", "remaining_time": "0:05:02"}
|
||||||
|
{"current_steps": 860, "total_steps": 940, "loss": 0.5846799373626709, "lr": 2.2448751396543788e-07, "epoch": 3.6613333333333333, "percentage": 91.49, "elapsed_time": "0:48:03", "remaining_time": "0:04:28"}
|
||||||
|
{"current_steps": 870, "total_steps": 940, "loss": 0.5573970794677734, "lr": 1.7278198177405614e-07, "epoch": 3.7039999999999997, "percentage": 92.55, "elapsed_time": "0:48:33", "remaining_time": "0:03:54"}
|
||||||
|
{"current_steps": 880, "total_steps": 940, "loss": 0.5604241371154786, "lr": 1.2773233789193816e-07, "epoch": 3.7466666666666666, "percentage": 93.62, "elapsed_time": "0:49:06", "remaining_time": "0:03:20"}
|
||||||
|
{"current_steps": 890, "total_steps": 940, "loss": 0.5811796188354492, "lr": 8.940069788894389e-08, "epoch": 3.7893333333333334, "percentage": 94.68, "elapsed_time": "0:49:39", "remaining_time": "0:02:47"}
|
||||||
|
{"current_steps": 900, "total_steps": 940, "loss": 0.5551981449127197, "lr": 5.783991438403802e-08, "epoch": 3.832, "percentage": 95.74, "elapsed_time": "0:50:13", "remaining_time": "0:02:13"}
|
||||||
|
{"current_steps": 910, "total_steps": 940, "loss": 0.5618691444396973, "lr": 3.309350417077972e-08, "epoch": 3.8746666666666667, "percentage": 96.81, "elapsed_time": "0:50:49", "remaining_time": "0:01:40"}
|
||||||
|
{"current_steps": 920, "total_steps": 940, "loss": 0.5391227722167968, "lr": 1.5195588215283773e-08, "epoch": 3.9173333333333336, "percentage": 97.87, "elapsed_time": "0:51:24", "remaining_time": "0:01:07"}
|
||||||
|
{"current_steps": 930, "total_steps": 940, "loss": 0.5671097278594971, "lr": 4.170844609387992e-09, "epoch": 3.96, "percentage": 98.94, "elapsed_time": "0:51:54", "remaining_time": "0:00:33"}
|
||||||
|
{"current_steps": 940, "total_steps": 940, "loss": 0.5210320949554443, "lr": 3.447454388127991e-11, "epoch": 4.0, "percentage": 100.0, "elapsed_time": "0:52:24", "remaining_time": "0:00:00"}
|
||||||
|
{"current_steps": 940, "total_steps": 940, "epoch": 4.0, "percentage": 100.0, "elapsed_time": "0:52:33", "remaining_time": "0:00:00"}
|
||||||
701
trainer_state.json
Normal file
701
trainer_state.json
Normal file
@@ -0,0 +1,701 @@
|
|||||||
|
{
|
||||||
|
"best_global_step": null,
|
||||||
|
"best_metric": null,
|
||||||
|
"best_model_checkpoint": null,
|
||||||
|
"epoch": 4.0,
|
||||||
|
"eval_steps": 500,
|
||||||
|
"global_step": 940,
|
||||||
|
"is_hyper_param_search": false,
|
||||||
|
"is_local_process_zero": true,
|
||||||
|
"is_world_process_zero": true,
|
||||||
|
"log_history": [
|
||||||
|
{
|
||||||
|
"epoch": 0.042666666666666665,
|
||||||
|
"grad_norm": 22.81595230102539,
|
||||||
|
"learning_rate": 9.574468085106384e-07,
|
||||||
|
"loss": 1.291497802734375,
|
||||||
|
"step": 10
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.08533333333333333,
|
||||||
|
"grad_norm": 8.337886810302734,
|
||||||
|
"learning_rate": 2.021276595744681e-06,
|
||||||
|
"loss": 1.182865810394287,
|
||||||
|
"step": 20
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.128,
|
||||||
|
"grad_norm": 2.478001594543457,
|
||||||
|
"learning_rate": 3.0851063829787237e-06,
|
||||||
|
"loss": 0.9575789451599122,
|
||||||
|
"step": 30
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.17066666666666666,
|
||||||
|
"grad_norm": 1.4279086589813232,
|
||||||
|
"learning_rate": 4.148936170212766e-06,
|
||||||
|
"loss": 0.8388131141662598,
|
||||||
|
"step": 40
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.21333333333333335,
|
||||||
|
"grad_norm": 1.040152907371521,
|
||||||
|
"learning_rate": 5.212765957446809e-06,
|
||||||
|
"loss": 0.8148186683654786,
|
||||||
|
"step": 50
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.256,
|
||||||
|
"grad_norm": 0.9204385876655579,
|
||||||
|
"learning_rate": 6.276595744680851e-06,
|
||||||
|
"loss": 0.7651220321655273,
|
||||||
|
"step": 60
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.2986666666666667,
|
||||||
|
"grad_norm": 0.8933783769607544,
|
||||||
|
"learning_rate": 7.340425531914894e-06,
|
||||||
|
"loss": 0.7899383544921875,
|
||||||
|
"step": 70
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.3413333333333333,
|
||||||
|
"grad_norm": 0.926196277141571,
|
||||||
|
"learning_rate": 8.404255319148937e-06,
|
||||||
|
"loss": 0.7782301902770996,
|
||||||
|
"step": 80
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.384,
|
||||||
|
"grad_norm": 0.9980058073997498,
|
||||||
|
"learning_rate": 9.46808510638298e-06,
|
||||||
|
"loss": 0.7684392929077148,
|
||||||
|
"step": 90
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.4266666666666667,
|
||||||
|
"grad_norm": 0.9042799472808838,
|
||||||
|
"learning_rate": 9.999138160172624e-06,
|
||||||
|
"loss": 0.7520565032958985,
|
||||||
|
"step": 100
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.4693333333333333,
|
||||||
|
"grad_norm": 0.8850747346878052,
|
||||||
|
"learning_rate": 9.99224522409411e-06,
|
||||||
|
"loss": 0.7115557670593262,
|
||||||
|
"step": 110
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.512,
|
||||||
|
"grad_norm": 0.9096830487251282,
|
||||||
|
"learning_rate": 9.97846885608885e-06,
|
||||||
|
"loss": 0.7434285163879395,
|
||||||
|
"step": 120
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.5546666666666666,
|
||||||
|
"grad_norm": 0.940818190574646,
|
||||||
|
"learning_rate": 9.957828051355817e-06,
|
||||||
|
"loss": 0.7457537651062012,
|
||||||
|
"step": 130
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.5973333333333334,
|
||||||
|
"grad_norm": 0.9445077180862427,
|
||||||
|
"learning_rate": 9.930351269950144e-06,
|
||||||
|
"loss": 0.7388599872589111,
|
||||||
|
"step": 140
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.64,
|
||||||
|
"grad_norm": 0.8463093042373657,
|
||||||
|
"learning_rate": 9.896076397541676e-06,
|
||||||
|
"loss": 0.7285007476806641,
|
||||||
|
"step": 150
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.6826666666666666,
|
||||||
|
"grad_norm": 0.8920971751213074,
|
||||||
|
"learning_rate": 9.855050693177286e-06,
|
||||||
|
"loss": 0.7659842491149902,
|
||||||
|
"step": 160
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.7253333333333334,
|
||||||
|
"grad_norm": 0.9197255969047546,
|
||||||
|
"learning_rate": 9.807330724118906e-06,
|
||||||
|
"loss": 0.724073839187622,
|
||||||
|
"step": 170
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.768,
|
||||||
|
"grad_norm": 0.9320898652076721,
|
||||||
|
"learning_rate": 9.752982287847193e-06,
|
||||||
|
"loss": 0.747947883605957,
|
||||||
|
"step": 180
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.8106666666666666,
|
||||||
|
"grad_norm": 0.8373203277587891,
|
||||||
|
"learning_rate": 9.692080321338317e-06,
|
||||||
|
"loss": 0.7283991813659668,
|
||||||
|
"step": 190
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.8533333333333334,
|
||||||
|
"grad_norm": 0.8440167307853699,
|
||||||
|
"learning_rate": 9.624708797739002e-06,
|
||||||
|
"loss": 0.7407590389251709,
|
||||||
|
"step": 200
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.896,
|
||||||
|
"grad_norm": 0.9070562720298767,
|
||||||
|
"learning_rate": 9.550960610582251e-06,
|
||||||
|
"loss": 0.724196481704712,
|
||||||
|
"step": 210
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.9386666666666666,
|
||||||
|
"grad_norm": 0.8882600665092468,
|
||||||
|
"learning_rate": 9.47093744570344e-06,
|
||||||
|
"loss": 0.7334166526794433,
|
||||||
|
"step": 220
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.9813333333333333,
|
||||||
|
"grad_norm": 0.9043309688568115,
|
||||||
|
"learning_rate": 9.384749641033358e-06,
|
||||||
|
"loss": 0.8169802665710449,
|
||||||
|
"step": 230
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 1.0213333333333334,
|
||||||
|
"grad_norm": 0.8862775564193726,
|
||||||
|
"learning_rate": 9.292516034461517e-06,
|
||||||
|
"loss": 0.7211981773376465,
|
||||||
|
"step": 240
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 1.064,
|
||||||
|
"grad_norm": 0.8914813995361328,
|
||||||
|
"learning_rate": 9.194363799979517e-06,
|
||||||
|
"loss": 0.6335953712463379,
|
||||||
|
"step": 250
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 1.1066666666666667,
|
||||||
|
"grad_norm": 0.9117757081985474,
|
||||||
|
"learning_rate": 9.090428272330381e-06,
|
||||||
|
"loss": 0.6689927577972412,
|
||||||
|
"step": 260
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 1.1493333333333333,
|
||||||
|
"grad_norm": 0.9108397364616394,
|
||||||
|
"learning_rate": 8.980852760405645e-06,
|
||||||
|
"loss": 0.6809295654296875,
|
||||||
|
"step": 270
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 1.192,
|
||||||
|
"grad_norm": 0.9220501780509949,
|
||||||
|
"learning_rate": 8.865788349647496e-06,
|
||||||
|
"loss": 0.6914172172546387,
|
||||||
|
"step": 280
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 1.2346666666666666,
|
||||||
|
"grad_norm": 0.8409727215766907,
|
||||||
|
"learning_rate": 8.745393693728395e-06,
|
||||||
|
"loss": 0.6650928974151611,
|
||||||
|
"step": 290
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 1.2773333333333334,
|
||||||
|
"grad_norm": 0.9512217044830322,
|
||||||
|
"learning_rate": 8.619834795795458e-06,
|
||||||
|
"loss": 0.6717124938964844,
|
||||||
|
"step": 300
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 1.32,
|
||||||
|
"grad_norm": 0.8877426385879517,
|
||||||
|
"learning_rate": 8.489284779581179e-06,
|
||||||
|
"loss": 0.6726479530334473,
|
||||||
|
"step": 310
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 1.3626666666666667,
|
||||||
|
"grad_norm": 0.8496021032333374,
|
||||||
|
"learning_rate": 8.353923650696119e-06,
|
||||||
|
"loss": 0.6797503471374512,
|
||||||
|
"step": 320
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 1.4053333333333333,
|
||||||
|
"grad_norm": 0.8745173811912537,
|
||||||
|
"learning_rate": 8.213938048432697e-06,
|
||||||
|
"loss": 0.6944191932678223,
|
||||||
|
"step": 330
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 1.448,
|
||||||
|
"grad_norm": 0.8742923736572266,
|
||||||
|
"learning_rate": 8.069520988422292e-06,
|
||||||
|
"loss": 0.619914960861206,
|
||||||
|
"step": 340
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 1.4906666666666666,
|
||||||
|
"grad_norm": 0.9417562484741211,
|
||||||
|
"learning_rate": 7.920871596500473e-06,
|
||||||
|
"loss": 0.6835647583007812,
|
||||||
|
"step": 350
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 1.5333333333333332,
|
||||||
|
"grad_norm": 0.9726133942604065,
|
||||||
|
"learning_rate": 7.768194834147362e-06,
|
||||||
|
"loss": 0.6706738471984863,
|
||||||
|
"step": 360
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 1.576,
|
||||||
|
"grad_norm": 0.8959471583366394,
|
||||||
|
"learning_rate": 7.611701215881635e-06,
|
||||||
|
"loss": 0.690638542175293,
|
||||||
|
"step": 370
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 1.6186666666666667,
|
||||||
|
"grad_norm": 0.9309572577476501,
|
||||||
|
"learning_rate": 7.4516065189978625e-06,
|
||||||
|
"loss": 0.6714137077331543,
|
||||||
|
"step": 380
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 1.6613333333333333,
|
||||||
|
"grad_norm": 0.9652783274650574,
|
||||||
|
"learning_rate": 7.288131486047414e-06,
|
||||||
|
"loss": 0.6779637336730957,
|
||||||
|
"step": 390
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 1.704,
|
||||||
|
"grad_norm": 0.8535190224647522,
|
||||||
|
"learning_rate": 7.121501520473137e-06,
|
||||||
|
"loss": 0.6494068622589111,
|
||||||
|
"step": 400
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 1.7466666666666666,
|
||||||
|
"grad_norm": 0.9693210124969482,
|
||||||
|
"learning_rate": 6.9519463758174745e-06,
|
||||||
|
"loss": 0.6467916011810303,
|
||||||
|
"step": 410
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 1.7893333333333334,
|
||||||
|
"grad_norm": 0.913555383682251,
|
||||||
|
"learning_rate": 6.77969983893257e-06,
|
||||||
|
"loss": 0.6865511894226074,
|
||||||
|
"step": 420
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 1.8319999999999999,
|
||||||
|
"grad_norm": 0.8626614212989807,
|
||||||
|
"learning_rate": 6.604999407629137e-06,
|
||||||
|
"loss": 0.6830032348632813,
|
||||||
|
"step": 430
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 1.8746666666666667,
|
||||||
|
"grad_norm": 0.9277188777923584,
|
||||||
|
"learning_rate": 6.428085963208567e-06,
|
||||||
|
"loss": 0.6937445640563965,
|
||||||
|
"step": 440
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 1.9173333333333333,
|
||||||
|
"grad_norm": 0.9122670292854309,
|
||||||
|
"learning_rate": 6.249203438329799e-06,
|
||||||
|
"loss": 0.6877000331878662,
|
||||||
|
"step": 450
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 1.96,
|
||||||
|
"grad_norm": 0.8714756369590759,
|
||||||
|
"learning_rate": 6.0685984806689055e-06,
|
||||||
|
"loss": 0.6529797077178955,
|
||||||
|
"step": 460
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 2.0,
|
||||||
|
"grad_norm": 1.4163620471954346,
|
||||||
|
"learning_rate": 5.886520112835128e-06,
|
||||||
|
"loss": 0.6860092163085938,
|
||||||
|
"step": 470
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 2.042666666666667,
|
||||||
|
"grad_norm": 0.8940126299858093,
|
||||||
|
"learning_rate": 5.703219389012317e-06,
|
||||||
|
"loss": 0.6075316905975342,
|
||||||
|
"step": 480
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 2.0853333333333333,
|
||||||
|
"grad_norm": 0.9413266181945801,
|
||||||
|
"learning_rate": 5.518949048799176e-06,
|
||||||
|
"loss": 0.5943183422088623,
|
||||||
|
"step": 490
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 2.128,
|
||||||
|
"grad_norm": 0.932360827922821,
|
||||||
|
"learning_rate": 5.3339631687256085e-06,
|
||||||
|
"loss": 0.5897200107574463,
|
||||||
|
"step": 500
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 2.1706666666666665,
|
||||||
|
"grad_norm": 0.9619448184967041,
|
||||||
|
"learning_rate": 5.148516811925684e-06,
|
||||||
|
"loss": 0.6106101036071777,
|
||||||
|
"step": 510
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 2.2133333333333334,
|
||||||
|
"grad_norm": 0.9640690684318542,
|
||||||
|
"learning_rate": 4.962865676450239e-06,
|
||||||
|
"loss": 0.6104888916015625,
|
||||||
|
"step": 520
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 2.2560000000000002,
|
||||||
|
"grad_norm": 0.9159106612205505,
|
||||||
|
"learning_rate": 4.777265742704039e-06,
|
||||||
|
"loss": 0.5798359870910644,
|
||||||
|
"step": 530
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 2.2986666666666666,
|
||||||
|
"grad_norm": 0.9406466484069824,
|
||||||
|
"learning_rate": 4.591972920493638e-06,
|
||||||
|
"loss": 0.6197110652923584,
|
||||||
|
"step": 540
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 2.3413333333333335,
|
||||||
|
"grad_norm": 1.0240126848220825,
|
||||||
|
"learning_rate": 4.40724269617256e-06,
|
||||||
|
"loss": 0.5831963539123535,
|
||||||
|
"step": 550
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 2.384,
|
||||||
|
"grad_norm": 0.9565466046333313,
|
||||||
|
"learning_rate": 4.223329780370359e-06,
|
||||||
|
"loss": 0.598298454284668,
|
||||||
|
"step": 560
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 2.4266666666666667,
|
||||||
|
"grad_norm": 0.9118269085884094,
|
||||||
|
"learning_rate": 4.04048775679127e-06,
|
||||||
|
"loss": 0.5884403705596923,
|
||||||
|
"step": 570
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 2.469333333333333,
|
||||||
|
"grad_norm": 0.9704470038414001,
|
||||||
|
"learning_rate": 3.858968732566685e-06,
|
||||||
|
"loss": 0.5839417457580567,
|
||||||
|
"step": 580
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 2.512,
|
||||||
|
"grad_norm": 0.949148416519165,
|
||||||
|
"learning_rate": 3.6790229906435706e-06,
|
||||||
|
"loss": 0.6246652126312255,
|
||||||
|
"step": 590
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 2.554666666666667,
|
||||||
|
"grad_norm": 0.9721353054046631,
|
||||||
|
"learning_rate": 3.5008986446881088e-06,
|
||||||
|
"loss": 0.6042469024658204,
|
||||||
|
"step": 600
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 2.5973333333333333,
|
||||||
|
"grad_norm": 1.11699378490448,
|
||||||
|
"learning_rate": 3.3248412969804065e-06,
|
||||||
|
"loss": 0.6285554885864257,
|
||||||
|
"step": 610
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 2.64,
|
||||||
|
"grad_norm": 1.023742437362671,
|
||||||
|
"learning_rate": 3.1510936997719557e-06,
|
||||||
|
"loss": 0.6006599426269531,
|
||||||
|
"step": 620
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 2.6826666666666665,
|
||||||
|
"grad_norm": 1.056766390800476,
|
||||||
|
"learning_rate": 2.9798954205727886e-06,
|
||||||
|
"loss": 0.5928638458251954,
|
||||||
|
"step": 630
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 2.7253333333333334,
|
||||||
|
"grad_norm": 1.0377728939056396,
|
||||||
|
"learning_rate": 2.811482511829842e-06,
|
||||||
|
"loss": 0.6103617668151855,
|
||||||
|
"step": 640
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 2.768,
|
||||||
|
"grad_norm": 0.9824387431144714,
|
||||||
|
"learning_rate": 2.6460871854519594e-06,
|
||||||
|
"loss": 0.5874536514282227,
|
||||||
|
"step": 650
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 2.8106666666666666,
|
||||||
|
"grad_norm": 1.0173842906951904,
|
||||||
|
"learning_rate": 2.483937492630345e-06,
|
||||||
|
"loss": 0.5812715053558349,
|
||||||
|
"step": 660
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 2.8533333333333335,
|
||||||
|
"grad_norm": 1.1885449886322021,
|
||||||
|
"learning_rate": 2.3252570093959e-06,
|
||||||
|
"loss": 0.609982681274414,
|
||||||
|
"step": 670
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 2.896,
|
||||||
|
"grad_norm": 0.974206805229187,
|
||||||
|
"learning_rate": 2.1702645283470238e-06,
|
||||||
|
"loss": 0.6121101379394531,
|
||||||
|
"step": 680
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 2.9386666666666668,
|
||||||
|
"grad_norm": 0.9060782194137573,
|
||||||
|
"learning_rate": 2.0191737569729492e-06,
|
||||||
|
"loss": 0.6171843528747558,
|
||||||
|
"step": 690
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 2.981333333333333,
|
||||||
|
"grad_norm": 1.0292282104492188,
|
||||||
|
"learning_rate": 1.872193022988526e-06,
|
||||||
|
"loss": 0.5880200386047363,
|
||||||
|
"step": 700
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 3.021333333333333,
|
||||||
|
"grad_norm": 1.00603449344635,
|
||||||
|
"learning_rate": 1.7295249870867898e-06,
|
||||||
|
"loss": 0.5795096397399903,
|
||||||
|
"step": 710
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 3.064,
|
||||||
|
"grad_norm": 0.9750069379806519,
|
||||||
|
"learning_rate": 1.5913663635053578e-06,
|
||||||
|
"loss": 0.5702518463134766,
|
||||||
|
"step": 720
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 3.1066666666666665,
|
||||||
|
"grad_norm": 0.9798346161842346,
|
||||||
|
"learning_rate": 1.457907648791943e-06,
|
||||||
|
"loss": 0.5839662551879883,
|
||||||
|
"step": 730
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 3.1493333333333333,
|
||||||
|
"grad_norm": 1.0039501190185547,
|
||||||
|
"learning_rate": 1.329332859142967e-06,
|
||||||
|
"loss": 0.5281820297241211,
|
||||||
|
"step": 740
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 3.192,
|
||||||
|
"grad_norm": 0.9754037857055664,
|
||||||
|
"learning_rate": 1.205819276677464e-06,
|
||||||
|
"loss": 0.5751385688781738,
|
||||||
|
"step": 750
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 3.2346666666666666,
|
||||||
|
"grad_norm": 0.990886926651001,
|
||||||
|
"learning_rate": 1.0875372049960697e-06,
|
||||||
|
"loss": 0.5494537353515625,
|
||||||
|
"step": 760
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 3.2773333333333334,
|
||||||
|
"grad_norm": 0.9957782626152039,
|
||||||
|
"learning_rate": 9.746497343621857e-07,
|
||||||
|
"loss": 0.5537368297576905,
|
||||||
|
"step": 770
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 3.32,
|
||||||
|
"grad_norm": 1.0468347072601318,
|
||||||
|
"learning_rate": 8.673125168290713e-07,
|
||||||
|
"loss": 0.55874662399292,
|
||||||
|
"step": 780
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 3.3626666666666667,
|
||||||
|
"grad_norm": 0.949479877948761,
|
||||||
|
"learning_rate": 7.656735516229125e-07,
|
||||||
|
"loss": 0.565284252166748,
|
||||||
|
"step": 790
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 3.405333333333333,
|
||||||
|
"grad_norm": 1.3505475521087646,
|
||||||
|
"learning_rate": 6.698729810778065e-07,
|
||||||
|
"loss": 0.5531235694885254,
|
||||||
|
"step": 800
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 3.448,
|
||||||
|
"grad_norm": 1.0080723762512207,
|
||||||
|
"learning_rate": 5.800428974040311e-07,
|
||||||
|
"loss": 0.5778326511383056,
|
||||||
|
"step": 810
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 3.490666666666667,
|
||||||
|
"grad_norm": 0.9772297143936157,
|
||||||
|
"learning_rate": 4.963071605560144e-07,
|
||||||
|
"loss": 0.5518184661865234,
|
||||||
|
"step": 820
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 3.533333333333333,
|
||||||
|
"grad_norm": 1.0470722913742065,
|
||||||
|
"learning_rate": 4.187812274511427e-07,
|
||||||
|
"loss": 0.5773732185363769,
|
||||||
|
"step": 830
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 3.576,
|
||||||
|
"grad_norm": 0.9107851982116699,
|
||||||
|
"learning_rate": 3.4757199277490106e-07,
|
||||||
|
"loss": 0.5663552284240723,
|
||||||
|
"step": 840
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 3.618666666666667,
|
||||||
|
"grad_norm": 0.9870362281799316,
|
||||||
|
"learning_rate": 2.8277764159181484e-07,
|
||||||
|
"loss": 0.5664983272552491,
|
||||||
|
"step": 850
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 3.6613333333333333,
|
||||||
|
"grad_norm": 1.0563737154006958,
|
||||||
|
"learning_rate": 2.2448751396543788e-07,
|
||||||
|
"loss": 0.5846799373626709,
|
||||||
|
"step": 860
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 3.7039999999999997,
|
||||||
|
"grad_norm": 1.0350754261016846,
|
||||||
|
"learning_rate": 1.7278198177405614e-07,
|
||||||
|
"loss": 0.5573970794677734,
|
||||||
|
"step": 870
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 3.7466666666666666,
|
||||||
|
"grad_norm": 1.0310237407684326,
|
||||||
|
"learning_rate": 1.2773233789193816e-07,
|
||||||
|
"loss": 0.5604241371154786,
|
||||||
|
"step": 880
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 3.7893333333333334,
|
||||||
|
"grad_norm": 0.9566333889961243,
|
||||||
|
"learning_rate": 8.940069788894389e-08,
|
||||||
|
"loss": 0.5811796188354492,
|
||||||
|
"step": 890
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 3.832,
|
||||||
|
"grad_norm": 0.9202328324317932,
|
||||||
|
"learning_rate": 5.783991438403802e-08,
|
||||||
|
"loss": 0.5551981449127197,
|
||||||
|
"step": 900
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 3.8746666666666667,
|
||||||
|
"grad_norm": 0.9610781073570251,
|
||||||
|
"learning_rate": 3.309350417077972e-08,
|
||||||
|
"loss": 0.5618691444396973,
|
||||||
|
"step": 910
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 3.9173333333333336,
|
||||||
|
"grad_norm": 0.9684038162231445,
|
||||||
|
"learning_rate": 1.5195588215283773e-08,
|
||||||
|
"loss": 0.5391227722167968,
|
||||||
|
"step": 920
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 3.96,
|
||||||
|
"grad_norm": 0.9603227972984314,
|
||||||
|
"learning_rate": 4.170844609387992e-09,
|
||||||
|
"loss": 0.5671097278594971,
|
||||||
|
"step": 930
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 4.0,
|
||||||
|
"grad_norm": 1.651139497756958,
|
||||||
|
"learning_rate": 3.447454388127991e-11,
|
||||||
|
"loss": 0.5210320949554443,
|
||||||
|
"step": 940
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 4.0,
|
||||||
|
"step": 940,
|
||||||
|
"total_flos": 7.974893111623025e+17,
|
||||||
|
"train_loss": 0.6604051219656112,
|
||||||
|
"train_runtime": 3155.399,
|
||||||
|
"train_samples_per_second": 38.03,
|
||||||
|
"train_steps_per_second": 0.298
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"logging_steps": 10,
|
||||||
|
"max_steps": 940,
|
||||||
|
"num_input_tokens_seen": 0,
|
||||||
|
"num_train_epochs": 4,
|
||||||
|
"save_steps": 500,
|
||||||
|
"stateful_callbacks": {
|
||||||
|
"TrainerControl": {
|
||||||
|
"args": {
|
||||||
|
"should_epoch_stop": false,
|
||||||
|
"should_evaluate": false,
|
||||||
|
"should_log": false,
|
||||||
|
"should_save": true,
|
||||||
|
"should_training_stop": true
|
||||||
|
},
|
||||||
|
"attributes": {}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"total_flos": 7.974893111623025e+17,
|
||||||
|
"train_batch_size": 2,
|
||||||
|
"trial_name": null,
|
||||||
|
"trial_params": null
|
||||||
|
}
|
||||||
3
training_args.bin
Normal file
3
training_args.bin
Normal file
@@ -0,0 +1,3 @@
|
|||||||
|
version https://git-lfs.github.com/spec/v1
|
||||||
|
oid sha256:26266069f0a3a139205f0115ae3df892adf8170151f2b6bb7f88e40771227ec5
|
||||||
|
size 7185
|
||||||
BIN
training_loss.png
Normal file
BIN
training_loss.png
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 39 KiB |
Reference in New Issue
Block a user