From 634c2e8d5761d82673df1aa258830d9f194759a2 Mon Sep 17 00:00:00 2001 From: ModelHub XC Date: Tue, 16 Jun 2026 05:05:17 +0800 Subject: [PATCH] =?UTF-8?q?=E5=88=9D=E5=A7=8B=E5=8C=96=E9=A1=B9=E7=9B=AE?= =?UTF-8?q?=EF=BC=8C=E7=94=B1ModelHub=20XC=E7=A4=BE=E5=8C=BA=E6=8F=90?= =?UTF-8?q?=E4=BE=9B=E6=A8=A1=E5=9E=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Model: KKHYA/qwen3-1.7b-fft-math Source: Original Platform --- .gitattributes | 36 + README.md | 61 + all_results.json | 8 + chat_template.jinja | 89 + config.json | 70 + generation_config.json | 12 + model.safetensors | 3 + qwen3_1.7b_math_fft_20260429_154203.log | 2843 +++++++++++++++++++++++ tokenizer.json | 3 + tokenizer_config.json | 30 + train_results.json | 8 + trainer_log.jsonl | 94 + trainer_state.json | 694 ++++++ training_args.bin | 3 + training_loss.png | Bin 0 -> 35377 bytes 15 files changed, 3954 insertions(+) create mode 100644 .gitattributes create mode 100644 README.md create mode 100644 all_results.json create mode 100644 chat_template.jinja create mode 100644 config.json create mode 100644 generation_config.json create mode 100644 model.safetensors create mode 100644 qwen3_1.7b_math_fft_20260429_154203.log create mode 100644 tokenizer.json create mode 100644 tokenizer_config.json create mode 100644 train_results.json create mode 100644 trainer_log.jsonl create mode 100644 trainer_state.json create mode 100644 training_args.bin create mode 100644 training_loss.png diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..52373fe --- /dev/null +++ b/.gitattributes @@ -0,0 +1,36 @@ +*.7z filter=lfs diff=lfs merge=lfs -text +*.arrow filter=lfs diff=lfs merge=lfs -text +*.bin filter=lfs diff=lfs merge=lfs -text +*.bz2 filter=lfs diff=lfs merge=lfs -text +*.ckpt filter=lfs diff=lfs merge=lfs -text +*.ftz filter=lfs diff=lfs merge=lfs -text +*.gz filter=lfs diff=lfs merge=lfs -text +*.h5 filter=lfs diff=lfs merge=lfs -text +*.joblib filter=lfs diff=lfs merge=lfs -text +*.lfs.* filter=lfs diff=lfs merge=lfs -text +*.mlmodel filter=lfs diff=lfs merge=lfs -text +*.model filter=lfs diff=lfs merge=lfs -text +*.msgpack filter=lfs diff=lfs merge=lfs -text +*.npy filter=lfs diff=lfs merge=lfs -text +*.npz filter=lfs diff=lfs merge=lfs -text +*.onnx filter=lfs diff=lfs merge=lfs -text +*.ot filter=lfs diff=lfs merge=lfs -text +*.parquet filter=lfs diff=lfs merge=lfs -text +*.pb filter=lfs diff=lfs merge=lfs -text +*.pickle filter=lfs diff=lfs merge=lfs -text +*.pkl filter=lfs diff=lfs merge=lfs -text +*.pt filter=lfs diff=lfs merge=lfs -text +*.pth filter=lfs diff=lfs merge=lfs -text +*.rar filter=lfs diff=lfs merge=lfs -text +*.safetensors filter=lfs diff=lfs merge=lfs -text +saved_model/**/* filter=lfs diff=lfs merge=lfs -text +*.tar.* filter=lfs diff=lfs merge=lfs -text +*.tar filter=lfs diff=lfs merge=lfs -text +*.tflite filter=lfs diff=lfs merge=lfs -text +*.tgz filter=lfs diff=lfs merge=lfs -text +*.wasm filter=lfs diff=lfs merge=lfs -text +*.xz filter=lfs diff=lfs merge=lfs -text +*.zip filter=lfs diff=lfs merge=lfs -text +*.zst filter=lfs diff=lfs merge=lfs -text +*tfevents* filter=lfs diff=lfs merge=lfs -text +tokenizer.json filter=lfs diff=lfs merge=lfs -text diff --git a/README.md b/README.md new file mode 100644 index 0000000..af77b3f --- /dev/null +++ b/README.md @@ -0,0 +1,61 @@ +--- +library_name: transformers +license: apache-2.0 +base_model: Qwen/Qwen3-1.7B +tags: +- llama-factory +- full +- generated_from_trainer +model-index: +- name: qwen3-1.7b-fft-math + results: [] +--- + + + +# qwen3-1.7b-fft-math + +This model is a fine-tuned version of [Qwen/Qwen3-1.7B](https://huggingface.co/Qwen/Qwen3-1.7B) on the mft_metamath, the mft_numinamath_tir, the mft_numinamath_cot, the mft_tulu3_personas_math, the mft_tulu3_personas_math_grade and the mft_tulu3_personas_algebra datasets. + +## Model description + +More information needed + +## Intended uses & limitations + +More information needed + +## Training and evaluation data + +More information needed + +## Training procedure + +### Training hyperparameters + +The following hyperparameters were used during training: +- learning_rate: 1e-05 +- train_batch_size: 2 +- eval_batch_size: 8 +- seed: 42 +- distributed_type: multi-GPU +- num_devices: 8 +- gradient_accumulation_steps: 8 +- total_train_batch_size: 128 +- total_eval_batch_size: 64 +- optimizer: Use OptimizerNames.ADAMW_TORCH_FUSED with betas=(0.9,0.95) and epsilon=1e-08 and optimizer_args=No additional optimizer arguments +- lr_scheduler_type: cosine +- lr_scheduler_warmup_steps: 0.1 +- num_epochs: 2.0 + +### Training results + + + +### Framework versions + +- Transformers 5.2.0 +- Pytorch 2.10.0+cu130 +- Datasets 4.0.0 +- Tokenizers 0.22.2 diff --git a/all_results.json b/all_results.json new file mode 100644 index 0000000..8064427 --- /dev/null +++ b/all_results.json @@ -0,0 +1,8 @@ +{ + "epoch": 2.0, + "total_flos": 1.2200945171646382e+18, + "train_loss": 0.2963919332032519, + "train_runtime": 3915.3376, + "train_samples_per_second": 30.649, + "train_steps_per_second": 0.24 +} \ No newline at end of file diff --git a/chat_template.jinja b/chat_template.jinja new file mode 100644 index 0000000..01be9b3 --- /dev/null +++ b/chat_template.jinja @@ -0,0 +1,89 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and message.content is string and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if message.content is string %} + {%- set content = message.content %} + {%- else %} + {%- set content = '' %} + {%- endif %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/config.json b/config.json new file mode 100644 index 0000000..e0b1310 --- /dev/null +++ b/config.json @@ -0,0 +1,70 @@ +{ + "architectures": [ + "Qwen3ForCausalLM" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "bos_token_id": null, + "dtype": "bfloat16", + "eos_token_id": 151645, + "head_dim": 128, + "hidden_act": "silu", + "hidden_size": 2048, + "initializer_range": 0.02, + "intermediate_size": 6144, + "layer_types": [ + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention" + ], + "masked_layers": null, + "max_position_embeddings": 40960, + "max_window_layers": 28, + "model_type": "qwen3", + "num_attention_heads": 16, + "num_hidden_layers": 28, + "num_key_value_heads": 8, + "pad_token_id": 151643, + "rms_norm_eps": 1e-06, + "rope_parameters": { + "rope_theta": 1000000, + "rope_type": "default" + }, + "sliding_window": null, + "sparsity_attn": null, + "sparsity_mlp": null, + "subnet_mode": null, + "subnet_type": null, + "threshold_attn": null, + "threshold_mlp": null, + "tie_word_embeddings": true, + "transformers_version": "5.2.0", + "use_cache": false, + "use_sliding_window": false, + "vocab_size": 151936 +} diff --git a/generation_config.json b/generation_config.json new file mode 100644 index 0000000..1701c94 --- /dev/null +++ b/generation_config.json @@ -0,0 +1,12 @@ +{ + "do_sample": true, + "eos_token_id": [ + 151645, + 151643 + ], + "pad_token_id": 151643, + "temperature": 0.6, + "top_k": 20, + "top_p": 0.95, + "transformers_version": "5.2.0" +} diff --git a/model.safetensors b/model.safetensors new file mode 100644 index 0000000..55075ec --- /dev/null +++ b/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:969ad94985c2c70a61c87a23ebf9c7028163014421cf750139e9a5197f25c82f +size 4063515640 diff --git a/qwen3_1.7b_math_fft_20260429_154203.log b/qwen3_1.7b_math_fft_20260429_154203.log new file mode 100644 index 0000000..919319c --- /dev/null +++ b/qwen3_1.7b_math_fft_20260429_154203.log @@ -0,0 +1,2843 @@ +==== STARTING EXPERIMENT: qwen3_1.7b_math_fft ==== +Log File: saves/qwen3_1.7b/math/fft/qwen3_1.7b_math_fft_20260429_154203.log +HF Hub: https://huggingface.co/KKHYA/qwen3-1.7b-fft-math +Timestamp: 2026-04-29 15:42:03 +===================================== +[INFO|2026-04-29 15:42:18] llamafactory.launcher:144 >> Initializing 8 distributed tasks at: 127.0.0.1:49311 +W0429 15:42:19.791000 2470856 site-packages/torch/distributed/run.py:852] +W0429 15:42:19.791000 2470856 site-packages/torch/distributed/run.py:852] ***************************************** +W0429 15:42:19.791000 2470856 site-packages/torch/distributed/run.py:852] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +W0429 15:42:19.791000 2470856 site-packages/torch/distributed/run.py:852] ***************************************** +warmup_ratio is deprecated and will be removed in v5.2. Use `warmup_steps` instead. +warmup_ratio is deprecated and will be removed in v5.2. Use `warmup_steps` instead. +warmup_ratio is deprecated and will be removed in v5.2. Use `warmup_steps` instead. +warmup_ratio is deprecated and will be removed in v5.2. Use `warmup_steps` instead. +warmup_ratio is deprecated and will be removed in v5.2. Use `warmup_steps` instead. +warmup_ratio is deprecated and will be removed in v5.2. Use `warmup_steps` instead. +warmup_ratio is deprecated and will be removed in v5.2. Use `warmup_steps` instead. +warmup_ratio is deprecated and will be removed in v5.2. Use `warmup_steps` instead. +df: /root/.triton/autotune: No such file or directory +df: df: /root/.triton/autotune/root/.triton/autotunedf: /root/.triton/autotunedf: df: /root/.triton/autotune/root/.triton/autotunedf: /root/.triton/autotune: No such file or directory +: No such file or directory +: No such file or directory +: No such file or directory +: No such file or directory +: No such file or directory +[W429 15:42:34.641839785 ProcessGroupNCCL.cpp:929] Warning: TORCH_NCCL_AVOID_RECORD_STREAMS is the default now, this environment variable is thus deprecated. (function operator()) +[W429 15:42:34.641839403 ProcessGroupNCCL.cpp:929] Warning: TORCH_NCCL_AVOID_RECORD_STREAMS is the default now, this environment variable is thus deprecated. (function operator()) +[W429 15:42:34.648204081 ProcessGroupNCCL.cpp:929] Warning: TORCH_NCCL_AVOID_RECORD_STREAMS is the default now, this environment variable is thus deprecated. (function operator()) +[W429 15:42:34.656834411 ProcessGroupNCCL.cpp:929] Warning: TORCH_NCCL_AVOID_RECORD_STREAMS is the default now, this environment variable is thus deprecated. (function operator()) +[W429 15:42:34.704861810 ProcessGroupNCCL.cpp:929] Warning: TORCH_NCCL_AVOID_RECORD_STREAMS is the default now, this environment variable is thus deprecated. (function operator()) +[W429 15:42:34.738333250 ProcessGroupNCCL.cpp:929] Warning: TORCH_NCCL_AVOID_RECORD_STREAMS is the default now, this environment variable is thus deprecated. (function operator()) +[W429 15:42:34.757560566 ProcessGroupNCCL.cpp:929] Warning: TORCH_NCCL_AVOID_RECORD_STREAMS is the default now, this environment variable is thus deprecated. (function operator()) +[W429 15:42:34.774695972 ProcessGroupNCCL.cpp:929] Warning: TORCH_NCCL_AVOID_RECORD_STREAMS is the default now, this environment variable is thus deprecated. (function operator()) +ywang29-p4d-debug-worker-0:2470947:2470947 [0] NCCL INFO ENV/Plugin: Could not find: libnccl-env.so +ywang29-p4d-debug-worker-0:2470947:2470947 [0] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth +ywang29-p4d-debug-worker-0:2470947:2470947 [0] NCCL INFO Bootstrap: Using eth0:10.200.162.206<0> +ywang29-p4d-debug-worker-0:2470947:2470947 [0] NCCL INFO cudaDriverVersion 13000 +ywang29-p4d-debug-worker-0:2470947:2470947 [0] NCCL INFO NCCL version 2.28.9+cuda13.0 +ywang29-p4d-debug-worker-0:2470955:2470955 [7] NCCL INFO ENV/Plugin: Could not find: libnccl-env.so +ywang29-p4d-debug-worker-0:2470954:2470954 [6] NCCL INFO ENV/Plugin: Could not find: libnccl-env.so +ywang29-p4d-debug-worker-0:2470955:2470955 [7] NCCL INFO cudaDriverVersion 13000 +ywang29-p4d-debug-worker-0:2470954:2470954 [6] NCCL INFO cudaDriverVersion 13000 +ywang29-p4d-debug-worker-0:2470954:2470954 [6] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth +ywang29-p4d-debug-worker-0:2470954:2470954 [6] NCCL INFO Bootstrap: Using eth0:10.200.162.206<0> +ywang29-p4d-debug-worker-0:2470954:2470954 [6] NCCL INFO NCCL version 2.28.9+cuda13.0 +ywang29-p4d-debug-worker-0:2470955:2470955 [7] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth +ywang29-p4d-debug-worker-0:2470955:2470955 [7] NCCL INFO Bootstrap: Using eth0:10.200.162.206<0> +ywang29-p4d-debug-worker-0:2470955:2470955 [7] NCCL INFO NCCL version 2.28.9+cuda13.0 +ywang29-p4d-debug-worker-0:2470948:2470948 [1] NCCL INFO ENV/Plugin: Could not find: libnccl-env.so +ywang29-p4d-debug-worker-0:2470948:2470948 [1] NCCL INFO cudaDriverVersion 13000 +ywang29-p4d-debug-worker-0:2470948:2470948 [1] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth +ywang29-p4d-debug-worker-0:2470948:2470948 [1] NCCL INFO Bootstrap: Using eth0:10.200.162.206<0> +ywang29-p4d-debug-worker-0:2470948:2470948 [1] NCCL INFO NCCL version 2.28.9+cuda13.0 +ywang29-p4d-debug-worker-0:2470955:2470955 [7] NCCL INFO Comm config Blocking set to 1 +ywang29-p4d-debug-worker-0:2470954:2470954 [6] NCCL INFO Comm config Blocking set to 1 +ywang29-p4d-debug-worker-0:2470947:2470947 [0] NCCL INFO Comm config Blocking set to 1 +ywang29-p4d-debug-worker-0:2470948:2470948 [1] NCCL INFO Comm config Blocking set to 1 +ywang29-p4d-debug-worker-0:2470951:2470951 [3] NCCL INFO ENV/Plugin: Could not find: libnccl-env.so +ywang29-p4d-debug-worker-0:2470951:2470951 [3] NCCL INFO cudaDriverVersion 13000 +ywang29-p4d-debug-worker-0:2470951:2470951 [3] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth +ywang29-p4d-debug-worker-0:2470951:2470951 [3] NCCL INFO Bootstrap: Using eth0:10.200.162.206<0> +ywang29-p4d-debug-worker-0:2470951:2470951 [3] NCCL INFO NCCL version 2.28.9+cuda13.0 +ywang29-p4d-debug-worker-0:2470952:2470952 [4] NCCL INFO ENV/Plugin: Could not find: libnccl-env.so +ywang29-p4d-debug-worker-0:2470952:2470952 [4] NCCL INFO cudaDriverVersion 13000 +ywang29-p4d-debug-worker-0:2470953:2470953 [5] NCCL INFO ENV/Plugin: Could not find: libnccl-env.so +ywang29-p4d-debug-worker-0:2470953:2470953 [5] NCCL INFO cudaDriverVersion 13000 +ywang29-p4d-debug-worker-0:2470952:2470952 [4] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth +ywang29-p4d-debug-worker-0:2470953:2470953 [5] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth +ywang29-p4d-debug-worker-0:2470952:2470952 [4] NCCL INFO Bootstrap: Using eth0:10.200.162.206<0> +ywang29-p4d-debug-worker-0:2470952:2470952 [4] NCCL INFO NCCL version 2.28.9+cuda13.0 +ywang29-p4d-debug-worker-0:2470951:2470951 [3] NCCL INFO Comm config Blocking set to 1 +ywang29-p4d-debug-worker-0:2470953:2470953 [5] NCCL INFO Bootstrap: Using eth0:10.200.162.206<0> +ywang29-p4d-debug-worker-0:2470953:2470953 [5] NCCL INFO NCCL version 2.28.9+cuda13.0 +ywang29-p4d-debug-worker-0:2470952:2470952 [4] NCCL INFO Comm config Blocking set to 1 +ywang29-p4d-debug-worker-0:2470953:2470953 [5] NCCL INFO Comm config Blocking set to 1 +ywang29-p4d-debug-worker-0:2470949:2470949 [2] NCCL INFO ENV/Plugin: Could not find: libnccl-env.so +ywang29-p4d-debug-worker-0:2470949:2470949 [2] NCCL INFO cudaDriverVersion 13000 +ywang29-p4d-debug-worker-0:2470949:2470949 [2] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth +ywang29-p4d-debug-worker-0:2470949:2470949 [2] NCCL INFO Bootstrap: Using eth0:10.200.162.206<0> +ywang29-p4d-debug-worker-0:2470949:2470949 [2] NCCL INFO NCCL version 2.28.9+cuda13.0 +ywang29-p4d-debug-worker-0:2470949:2470949 [2] NCCL INFO Comm config Blocking set to 1 +ywang29-p4d-debug-worker-0:2470948:2470948 [1] NCCL INFO NET/Plugin: Loaded net plugin Libfabric (v10) +ywang29-p4d-debug-worker-0:2470948:2470948 [1] NCCL INFO Successfully loaded external network plugin /opt/amazon/ofi-nccl/lib/libnccl-net.so +ywang29-p4d-debug-worker-0:2470948:2470948 [1] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth +ywang29-p4d-debug-worker-0:2470954:2470954 [6] NCCL INFO NET/Plugin: Loaded net plugin Libfabric (v10) +ywang29-p4d-debug-worker-0:2470954:2470954 [6] NCCL INFO Successfully loaded external network plugin /opt/amazon/ofi-nccl/lib/libnccl-net.so +ywang29-p4d-debug-worker-0:2470954:2470954 [6] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth +ywang29-p4d-debug-worker-0:2470948:2470948 [1] NCCL INFO NET/Socket : Using [0]eth0:10.200.162.206<0> +ywang29-p4d-debug-worker-0:2470948:2470948 [1] NCCL INFO Initialized NET plugin Socket +ywang29-p4d-debug-worker-0:2470954:2470954 [6] NCCL INFO NET/Socket : Using [0]eth0:10.200.162.206<0> +ywang29-p4d-debug-worker-0:2470954:2470954 [6] NCCL INFO Initialized NET plugin Socket +ywang29-p4d-debug-worker-0:2470948:2470948 [1] NCCL INFO Assigned NET plugin Socket to comm +ywang29-p4d-debug-worker-0:2470948:2470948 [1] NCCL INFO Using network Socket +ywang29-p4d-debug-worker-0:2470954:2470954 [6] NCCL INFO Assigned NET plugin Socket to comm +ywang29-p4d-debug-worker-0:2470954:2470954 [6] NCCL INFO Using network Socket +ywang29-p4d-debug-worker-0:2470955:2470955 [7] NCCL INFO NET/Plugin: Loaded net plugin Libfabric (v10) +ywang29-p4d-debug-worker-0:2470955:2470955 [7] NCCL INFO Successfully loaded external network plugin /opt/amazon/ofi-nccl/lib/libnccl-net.so +ywang29-p4d-debug-worker-0:2470955:2470955 [7] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth +ywang29-p4d-debug-worker-0:2470955:2470955 [7] NCCL INFO NET/Socket : Using [0]eth0:10.200.162.206<0> +ywang29-p4d-debug-worker-0:2470955:2470955 [7] NCCL INFO Initialized NET plugin Socket +ywang29-p4d-debug-worker-0:2470955:2470955 [7] NCCL INFO Assigned NET plugin Socket to comm +ywang29-p4d-debug-worker-0:2470955:2470955 [7] NCCL INFO Using network Socket +ywang29-p4d-debug-worker-0:2470947:2470947 [0] NCCL INFO NET/Plugin: Loaded net plugin Libfabric (v10) +ywang29-p4d-debug-worker-0:2470947:2470947 [0] NCCL INFO Successfully loaded external network plugin /opt/amazon/ofi-nccl/lib/libnccl-net.so +ywang29-p4d-debug-worker-0:2470947:2470947 [0] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth +ywang29-p4d-debug-worker-0:2470947:2470947 [0] NCCL INFO NET/Socket : Using [0]eth0:10.200.162.206<0> +ywang29-p4d-debug-worker-0:2470947:2470947 [0] NCCL INFO Initialized NET plugin Socket +ywang29-p4d-debug-worker-0:2470947:2470947 [0] NCCL INFO Assigned NET plugin Socket to comm +ywang29-p4d-debug-worker-0:2470947:2470947 [0] NCCL INFO Using network Socket +ywang29-p4d-debug-worker-0:2470951:2470951 [3] NCCL INFO NET/Plugin: Loaded net plugin Libfabric (v10) +ywang29-p4d-debug-worker-0:2470951:2470951 [3] NCCL INFO Successfully loaded external network plugin /opt/amazon/ofi-nccl/lib/libnccl-net.so +ywang29-p4d-debug-worker-0:2470951:2470951 [3] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth +ywang29-p4d-debug-worker-0:2470951:2470951 [3] NCCL INFO NET/Socket : Using [0]eth0:10.200.162.206<0> +ywang29-p4d-debug-worker-0:2470951:2470951 [3] NCCL INFO Initialized NET plugin Socket +ywang29-p4d-debug-worker-0:2470951:2470951 [3] NCCL INFO Assigned NET plugin Socket to comm +ywang29-p4d-debug-worker-0:2470951:2470951 [3] NCCL INFO Using network Socket +ywang29-p4d-debug-worker-0:2470952:2470952 [4] NCCL INFO NET/Plugin: Loaded net plugin Libfabric (v10) +ywang29-p4d-debug-worker-0:2470952:2470952 [4] NCCL INFO Successfully loaded external network plugin /opt/amazon/ofi-nccl/lib/libnccl-net.so +ywang29-p4d-debug-worker-0:2470952:2470952 [4] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth +ywang29-p4d-debug-worker-0:2470952:2470952 [4] NCCL INFO NET/Socket : Using [0]eth0:10.200.162.206<0> +ywang29-p4d-debug-worker-0:2470952:2470952 [4] NCCL INFO Initialized NET plugin Socket +ywang29-p4d-debug-worker-0:2470952:2470952 [4] NCCL INFO Assigned NET plugin Socket to comm +ywang29-p4d-debug-worker-0:2470952:2470952 [4] NCCL INFO Using network Socket +ywang29-p4d-debug-worker-0:2470953:2470953 [5] NCCL INFO NET/Plugin: Loaded net plugin Libfabric (v10) +ywang29-p4d-debug-worker-0:2470953:2470953 [5] NCCL INFO Successfully loaded external network plugin /opt/amazon/ofi-nccl/lib/libnccl-net.so +ywang29-p4d-debug-worker-0:2470953:2470953 [5] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth +ywang29-p4d-debug-worker-0:2470953:2470953 [5] NCCL INFO NET/Socket : Using [0]eth0:10.200.162.206<0> +ywang29-p4d-debug-worker-0:2470953:2470953 [5] NCCL INFO Initialized NET plugin Socket +ywang29-p4d-debug-worker-0:2470953:2470953 [5] NCCL INFO Assigned NET plugin Socket to comm +ywang29-p4d-debug-worker-0:2470953:2470953 [5] NCCL INFO Using network Socket +ywang29-p4d-debug-worker-0:2470949:2470949 [2] NCCL INFO NET/Plugin: Loaded net plugin Libfabric (v10) +ywang29-p4d-debug-worker-0:2470949:2470949 [2] NCCL INFO Successfully loaded external network plugin /opt/amazon/ofi-nccl/lib/libnccl-net.so +ywang29-p4d-debug-worker-0:2470949:2470949 [2] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth +ywang29-p4d-debug-worker-0:2470949:2470949 [2] NCCL INFO NET/Socket : Using [0]eth0:10.200.162.206<0> +ywang29-p4d-debug-worker-0:2470949:2470949 [2] NCCL INFO Initialized NET plugin Socket +ywang29-p4d-debug-worker-0:2470949:2470949 [2] NCCL INFO Assigned NET plugin Socket to comm +ywang29-p4d-debug-worker-0:2470949:2470949 [2] NCCL INFO Using network Socket +ywang29-p4d-debug-worker-0:2470955:2470955 [7] NCCL INFO ncclCommInitRankConfig comm 0x55e36d363080 rank 7 nranks 8 cudaDev 7 nvmlDev 7 busId a01d0 commId 0xbdc52b61bae6206d - Init START +ywang29-p4d-debug-worker-0:2470954:2470954 [6] NCCL INFO ncclCommInitRankConfig comm 0x56071a5c6320 rank 6 nranks 8 cudaDev 6 nvmlDev 6 busId a01c0 commId 0xbdc52b61bae6206d - Init START +ywang29-p4d-debug-worker-0:2470948:2470948 [1] NCCL INFO ncclCommInitRankConfig comm 0x558cf7486a10 rank 1 nranks 8 cudaDev 1 nvmlDev 1 busId 101d0 commId 0xbdc52b61bae6206d - Init START +ywang29-p4d-debug-worker-0:2470947:2470947 [0] NCCL INFO ncclCommInitRankConfig comm 0x55ffbe0f7c80 rank 0 nranks 8 cudaDev 0 nvmlDev 0 busId 101c0 commId 0xbdc52b61bae6206d - Init START +ywang29-p4d-debug-worker-0:2470955:2470955 [7] NCCL INFO RAS client listening socket at ::1<28028> +ywang29-p4d-debug-worker-0:2470947:2470947 [0] NCCL INFO RAS client listening socket at ::1<28028> +ywang29-p4d-debug-worker-0:2470952:2470952 [4] NCCL INFO ncclCommInitRankConfig comm 0x5647078795e0 rank 4 nranks 8 cudaDev 4 nvmlDev 4 busId 901c0 commId 0xbdc52b61bae6206d - Init START +ywang29-p4d-debug-worker-0:2470951:2470951 [3] NCCL INFO ncclCommInitRankConfig comm 0x5612d38dcbf0 rank 3 nranks 8 cudaDev 3 nvmlDev 3 busId 201d0 commId 0xbdc52b61bae6206d - Init START +ywang29-p4d-debug-worker-0:2470949:2470949 [2] NCCL INFO ncclCommInitRankConfig comm 0x5602809d0750 rank 2 nranks 8 cudaDev 2 nvmlDev 2 busId 201c0 commId 0xbdc52b61bae6206d - Init START +ywang29-p4d-debug-worker-0:2470953:2470953 [5] NCCL INFO ncclCommInitRankConfig comm 0x55c0201d0a60 rank 5 nranks 8 cudaDev 5 nvmlDev 5 busId 901d0 commId 0xbdc52b61bae6206d - Init START +ywang29-p4d-debug-worker-0:2470949:2470949 [2] NCCL INFO RAS client listening socket at ::1<28028> +ywang29-p4d-debug-worker-0:2470948:2470948 [1] NCCL INFO RAS client listening socket at ::1<28028> +ywang29-p4d-debug-worker-0:2470951:2470951 [3] NCCL INFO RAS client listening socket at ::1<28028> +ywang29-p4d-debug-worker-0:2470952:2470952 [4] NCCL INFO RAS client listening socket at ::1<28028> +ywang29-p4d-debug-worker-0:2470953:2470953 [5] NCCL INFO RAS client listening socket at ::1<28028> +ywang29-p4d-debug-worker-0:2470954:2470954 [6] NCCL INFO RAS client listening socket at ::1<28028> +ywang29-p4d-debug-worker-0:2470948:2470948 [1] NCCL INFO Bootstrap timings total 0.008871 (create 0.000085, send 0.000076, recv 0.007974, ring 0.000343, delay 0.000001) +ywang29-p4d-debug-worker-0:2470949:2470949 [2] NCCL INFO Bootstrap timings total 0.001031 (create 0.000050, send 0.000072, recv 0.000209, ring 0.000343, delay 0.000001) +ywang29-p4d-debug-worker-0:2470951:2470951 [3] NCCL INFO Bootstrap timings total 0.001899 (create 0.000034, send 0.000071, recv 0.000096, ring 0.000324, delay 0.000001) +ywang29-p4d-debug-worker-0:2470947:2470947 [0] NCCL INFO Bootstrap timings total 0.005427 (create 0.000034, send 0.000064, recv 0.000224, ring 0.004700, delay 0.000001) +ywang29-p4d-debug-worker-0:2470952:2470952 [4] NCCL INFO Bootstrap timings total 0.002049 (create 0.000036, send 0.000068, recv 0.001328, ring 0.000239, delay 0.000001) +ywang29-p4d-debug-worker-0:2470955:2470955 [7] NCCL INFO Bootstrap timings total 0.029934 (create 0.000045, send 0.000090, recv 0.024643, ring 0.004705, delay 0.000001) +ywang29-p4d-debug-worker-0:2470953:2470953 [5] NCCL INFO Bootstrap timings total 0.000897 (create 0.000036, send 0.000064, recv 0.000252, ring 0.000131, delay 0.000001) +ywang29-p4d-debug-worker-0:2470954:2470954 [6] NCCL INFO Bootstrap timings total 0.010645 (create 0.000037, send 0.000077, recv 0.000107, ring 0.000122, delay 0.000001) +ywang29-p4d-debug-worker-0:2470951:2470951 [3] NCCL INFO ncclTopoGetCpuAffinity: Affinity for GPU 3 is 0-23,48-71. (GPU affinity = 0-23,48-71 ; CPU affinity = 0-95). +ywang29-p4d-debug-worker-0:2470951:2470951 [3] NCCL INFO NVLS multicast support is not available on dev 3 (NVLS_NCHANNELS 0) +ywang29-p4d-debug-worker-0:2470952:2470952 [4] NCCL INFO ncclTopoGetCpuAffinity: Affinity for GPU 4 is 24-47,72-95. (GPU affinity = 24-47,72-95 ; CPU affinity = 0-95). +ywang29-p4d-debug-worker-0:2470952:2470952 [4] NCCL INFO NVLS multicast support is not available on dev 4 (NVLS_NCHANNELS 0) +ywang29-p4d-debug-worker-0:2470949:2470949 [2] NCCL INFO ncclTopoGetCpuAffinity: Affinity for GPU 2 is 0-23,48-71. (GPU affinity = 0-23,48-71 ; CPU affinity = 0-95). +ywang29-p4d-debug-worker-0:2470949:2470949 [2] NCCL INFO NVLS multicast support is not available on dev 2 (NVLS_NCHANNELS 0) +ywang29-p4d-debug-worker-0:2470948:2470948 [1] NCCL INFO ncclTopoGetCpuAffinity: Affinity for GPU 1 is 0-23,48-71. (GPU affinity = 0-23,48-71 ; CPU affinity = 0-95). +ywang29-p4d-debug-worker-0:2470948:2470948 [1] NCCL INFO NVLS multicast support is not available on dev 1 (NVLS_NCHANNELS 0) +ywang29-p4d-debug-worker-0:2470953:2470953 [5] NCCL INFO ncclTopoGetCpuAffinity: Affinity for GPU 5 is 24-47,72-95. (GPU affinity = 24-47,72-95 ; CPU affinity = 0-95). +ywang29-p4d-debug-worker-0:2470953:2470953 [5] NCCL INFO NVLS multicast support is not available on dev 5 (NVLS_NCHANNELS 0) +ywang29-p4d-debug-worker-0:2470954:2470954 [6] NCCL INFO ncclTopoGetCpuAffinity: Affinity for GPU 6 is 24-47,72-95. (GPU affinity = 24-47,72-95 ; CPU affinity = 0-95). +ywang29-p4d-debug-worker-0:2470954:2470954 [6] NCCL INFO NVLS multicast support is not available on dev 6 (NVLS_NCHANNELS 0) +ywang29-p4d-debug-worker-0:2470947:2470947 [0] NCCL INFO ncclTopoGetCpuAffinity: Affinity for GPU 0 is 0-23,48-71. (GPU affinity = 0-23,48-71 ; CPU affinity = 0-95). +ywang29-p4d-debug-worker-0:2470947:2470947 [0] NCCL INFO NVLS multicast support is not available on dev 0 (NVLS_NCHANNELS 0) +ywang29-p4d-debug-worker-0:2470955:2470955 [7] NCCL INFO ncclTopoGetCpuAffinity: Affinity for GPU 7 is 24-47,72-95. (GPU affinity = 24-47,72-95 ; CPU affinity = 0-95). +ywang29-p4d-debug-worker-0:2470955:2470955 [7] NCCL INFO NVLS multicast support is not available on dev 7 (NVLS_NCHANNELS 0) +ywang29-p4d-debug-worker-0:2470951:2470951 [3] NCCL INFO comm 0x5612d38dcbf0 rank 3 nRanks 8 nNodes 1 localRanks 8 localRank 3 MNNVL 0 +ywang29-p4d-debug-worker-0:2470953:2470953 [5] NCCL INFO comm 0x55c0201d0a60 rank 5 nRanks 8 nNodes 1 localRanks 8 localRank 5 MNNVL 0 +ywang29-p4d-debug-worker-0:2470952:2470952 [4] NCCL INFO comm 0x5647078795e0 rank 4 nRanks 8 nNodes 1 localRanks 8 localRank 4 MNNVL 0 +ywang29-p4d-debug-worker-0:2470954:2470954 [6] NCCL INFO comm 0x56071a5c6320 rank 6 nRanks 8 nNodes 1 localRanks 8 localRank 6 MNNVL 0 +ywang29-p4d-debug-worker-0:2470948:2470948 [1] NCCL INFO comm 0x558cf7486a10 rank 1 nRanks 8 nNodes 1 localRanks 8 localRank 1 MNNVL 0 +ywang29-p4d-debug-worker-0:2470955:2470955 [7] NCCL INFO comm 0x55e36d363080 rank 7 nRanks 8 nNodes 1 localRanks 8 localRank 7 MNNVL 0 +ywang29-p4d-debug-worker-0:2470949:2470949 [2] NCCL INFO comm 0x5602809d0750 rank 2 nRanks 8 nNodes 1 localRanks 8 localRank 2 MNNVL 0 +ywang29-p4d-debug-worker-0:2470947:2470947 [0] NCCL INFO comm 0x55ffbe0f7c80 rank 0 nRanks 8 nNodes 1 localRanks 8 localRank 0 MNNVL 0 +ywang29-p4d-debug-worker-0:2470947:2470947 [0] NCCL INFO Channel 00/24 : 0 1 2 3 4 5 6 7 +ywang29-p4d-debug-worker-0:2470947:2470947 [0] NCCL INFO Channel 01/24 : 0 1 2 3 4 5 6 7 +ywang29-p4d-debug-worker-0:2470947:2470947 [0] NCCL INFO Channel 02/24 : 0 1 2 3 4 5 6 7 +ywang29-p4d-debug-worker-0:2470947:2470947 [0] NCCL INFO Channel 03/24 : 0 1 2 3 4 5 6 7 +ywang29-p4d-debug-worker-0:2470954:2470954 [6] NCCL INFO Trees [0] 7/-1/-1->6->5 [1] 7/-1/-1->6->5 [2] 7/-1/-1->6->5 [3] 7/-1/-1->6->5 [4] 7/-1/-1->6->5 [5] 7/-1/-1->6->5 [6] 7/-1/-1->6->5 [7] 7/-1/-1->6->5 [8] 7/-1/-1->6->5 [9] 7/-1/-1->6->5 [10] 7/-1/-1->6->5 [11] 7/-1/-1->6->5 [12] 7/-1/-1->6->5 [13] 7/-1/-1->6->5 [14] 7/-1/-1->6->5 [15] 7/-1/-1->6->5 [16] 7/-1/-1->6->5 [17] 7/-1/-1->6->5 [18] 7/-1/-1->6->5 [19] 7/-1/-1->6->5 [20] 7/-1/-1->6->5 [21] 7/-1/-1->6->5 [22] 7/-1/-1->6->5 [23] 7/-1/-1->6->5 +ywang29-p4d-debug-worker-0:2470953:2470953 [5] NCCL INFO Trees [0] 6/-1/-1->5->4 [1] 6/-1/-1->5->4 [2] 6/-1/-1->5->4 [3] 6/-1/-1->5->4 [4] 6/-1/-1->5->4 [5] 6/-1/-1->5->4 [6] 6/-1/-1->5->4 [7] 6/-1/-1->5->4 [8] 6/-1/-1->5->4 [9] 6/-1/-1->5->4 [10] 6/-1/-1->5->4 [11] 6/-1/-1->5->4 [12] 6/-1/-1->5->4 [13] 6/-1/-1->5->4 [14] 6/-1/-1->5->4 [15] 6/-1/-1->5->4 [16] 6/-1/-1->5->4 [17] 6/-1/-1->5->4 [18] 6/-1/-1->5->4 [19] 6/-1/-1->5->4 [20] 6/-1/-1->5->4 [21] 6/-1/-1->5->4 [22] 6/-1/-1->5->4 [23] 6/-1/-1->5->4 +ywang29-p4d-debug-worker-0:2470952:2470952 [4] NCCL INFO Trees [0] 5/-1/-1->4->3 [1] 5/-1/-1->4->3 [2] 5/-1/-1->4->3 [3] 5/-1/-1->4->3 [4] 5/-1/-1->4->3 [5] 5/-1/-1->4->3 [6] 5/-1/-1->4->3 [7] 5/-1/-1->4->3 [8] 5/-1/-1->4->3 [9] 5/-1/-1->4->3 [10] 5/-1/-1->4->3 [11] 5/-1/-1->4->3 [12] 5/-1/-1->4->3 [13] 5/-1/-1->4->3 [14] 5/-1/-1->4->3 [15] 5/-1/-1->4->3 [16] 5/-1/-1->4->3 [17] 5/-1/-1->4->3 [18] 5/-1/-1->4->3 [19] 5/-1/-1->4->3 [20] 5/-1/-1->4->3 [21] 5/-1/-1->4->3 [22] 5/-1/-1->4->3 [23] 5/-1/-1->4->3 +ywang29-p4d-debug-worker-0:2470947:2470947 [0] NCCL INFO Channel 04/24 : 0 1 2 3 4 5 6 7 +ywang29-p4d-debug-worker-0:2470951:2470951 [3] NCCL INFO Trees [0] 4/-1/-1->3->2 [1] 4/-1/-1->3->2 [2] 4/-1/-1->3->2 [3] 4/-1/-1->3->2 [4] 4/-1/-1->3->2 [5] 4/-1/-1->3->2 [6] 4/-1/-1->3->2 [7] 4/-1/-1->3->2 [8] 4/-1/-1->3->2 [9] 4/-1/-1->3->2 [10] 4/-1/-1->3->2 [11] 4/-1/-1->3->2 [12] 4/-1/-1->3->2 [13] 4/-1/-1->3->2 [14] 4/-1/-1->3->2 [15] 4/-1/-1->3->2 [16] 4/-1/-1->3->2 [17] 4/-1/-1->3->2 [18] 4/-1/-1->3->2 [19] 4/-1/-1->3->2 [20] 4/-1/-1->3->2 [21] 4/-1/-1->3->2 [22] 4/-1/-1->3->2 [23] 4/-1/-1->3->2 +ywang29-p4d-debug-worker-0:2470947:2470947 [0] NCCL INFO Channel 05/24 : 0 1 2 3 4 5 6 7 +ywang29-p4d-debug-worker-0:2470955:2470955 [7] NCCL INFO Trees [0] -1/-1/-1->7->6 [1] -1/-1/-1->7->6 [2] -1/-1/-1->7->6 [3] -1/-1/-1->7->6 [4] -1/-1/-1->7->6 [5] -1/-1/-1->7->6 [6] -1/-1/-1->7->6 [7] -1/-1/-1->7->6 [8] -1/-1/-1->7->6 [9] -1/-1/-1->7->6 [10] -1/-1/-1->7->6 [11] -1/-1/-1->7->6 [12] -1/-1/-1->7->6 [13] -1/-1/-1->7->6 [14] -1/-1/-1->7->6 [15] -1/-1/-1->7->6 [16] -1/-1/-1->7->6 [17] -1/-1/-1->7->6 [18] -1/-1/-1->7->6 [19] -1/-1/-1->7->6 [20] -1/-1/-1->7->6 [21] -1/-1/-1->7->6 [22] -1/-1/-1->7->6 [23] -1/-1/-1->7->6 +ywang29-p4d-debug-worker-0:2470954:2470954 [6] NCCL INFO P2P Chunksize set to 524288 +ywang29-p4d-debug-worker-0:2470953:2470953 [5] NCCL INFO P2P Chunksize set to 524288 +ywang29-p4d-debug-worker-0:2470949:2470949 [2] NCCL INFO Trees [0] 3/-1/-1->2->1 [1] 3/-1/-1->2->1 [2] 3/-1/-1->2->1 [3] 3/-1/-1->2->1 [4] 3/-1/-1->2->1 [5] 3/-1/-1->2->1 [6] 3/-1/-1->2->1 [7] 3/-1/-1->2->1 [8] 3/-1/-1->2->1 [9] 3/-1/-1->2->1 [10] 3/-1/-1->2->1 [11] 3/-1/-1->2->1 [12] 3/-1/-1->2->1 [13] 3/-1/-1->2->1 [14] 3/-1/-1->2->1 [15] 3/-1/-1->2->1 [16] 3/-1/-1->2->1 [17] 3/-1/-1->2->1 [18] 3/-1/-1->2->1 [19] 3/-1/-1->2->1 [20] 3/-1/-1->2->1 [21] 3/-1/-1->2->1 [22] 3/-1/-1->2->1 [23] 3/-1/-1->2->1 +ywang29-p4d-debug-worker-0:2470947:2470947 [0] NCCL INFO Channel 06/24 : 0 1 2 3 4 5 6 7 +ywang29-p4d-debug-worker-0:2470952:2470952 [4] NCCL INFO P2P Chunksize set to 524288 +ywang29-p4d-debug-worker-0:2470951:2470951 [3] NCCL INFO P2P Chunksize set to 524288 +ywang29-p4d-debug-worker-0:2470948:2470948 [1] NCCL INFO Trees [0] 2/-1/-1->1->0 [1] 2/-1/-1->1->0 [2] 2/-1/-1->1->0 [3] 2/-1/-1->1->0 [4] 2/-1/-1->1->0 [5] 2/-1/-1->1->0 [6] 2/-1/-1->1->0 [7] 2/-1/-1->1->0 [8] 2/-1/-1->1->0 [9] 2/-1/-1->1->0 [10] 2/-1/-1->1->0 [11] 2/-1/-1->1->0 [12] 2/-1/-1->1->0 [13] 2/-1/-1->1->0 [14] 2/-1/-1->1->0 [15] 2/-1/-1->1->0 [16] 2/-1/-1->1->0 [17] 2/-1/-1->1->0 [18] 2/-1/-1->1->0 [19] 2/-1/-1->1->0 [20] 2/-1/-1->1->0 [21] 2/-1/-1->1->0 [22] 2/-1/-1->1->0 [23] 2/-1/-1->1->0 +ywang29-p4d-debug-worker-0:2470947:2470947 [0] NCCL INFO Channel 07/24 : 0 1 2 3 4 5 6 7 +ywang29-p4d-debug-worker-0:2470955:2470955 [7] NCCL INFO P2P Chunksize set to 524288 +ywang29-p4d-debug-worker-0:2470949:2470949 [2] NCCL INFO P2P Chunksize set to 524288 +ywang29-p4d-debug-worker-0:2470947:2470947 [0] NCCL INFO Channel 08/24 : 0 1 2 3 4 5 6 7 +ywang29-p4d-debug-worker-0:2470947:2470947 [0] NCCL INFO Channel 09/24 : 0 1 2 3 4 5 6 7 +ywang29-p4d-debug-worker-0:2470948:2470948 [1] NCCL INFO P2P Chunksize set to 524288 +ywang29-p4d-debug-worker-0:2470947:2470947 [0] NCCL INFO Channel 10/24 : 0 1 2 3 4 5 6 7 +ywang29-p4d-debug-worker-0:2470947:2470947 [0] NCCL INFO Channel 11/24 : 0 1 2 3 4 5 6 7 +ywang29-p4d-debug-worker-0:2470947:2470947 [0] NCCL INFO Channel 12/24 : 0 1 2 3 4 5 6 7 +ywang29-p4d-debug-worker-0:2470947:2470947 [0] NCCL INFO Channel 13/24 : 0 1 2 3 4 5 6 7 +ywang29-p4d-debug-worker-0:2470947:2470947 [0] NCCL INFO Channel 14/24 : 0 1 2 3 4 5 6 7 +ywang29-p4d-debug-worker-0:2470947:2470947 [0] NCCL INFO Channel 15/24 : 0 1 2 3 4 5 6 7 +ywang29-p4d-debug-worker-0:2470947:2470947 [0] NCCL INFO Channel 16/24 : 0 1 2 3 4 5 6 7 +ywang29-p4d-debug-worker-0:2470947:2470947 [0] NCCL INFO Channel 17/24 : 0 1 2 3 4 5 6 7 +ywang29-p4d-debug-worker-0:2470947:2470947 [0] NCCL INFO Channel 18/24 : 0 1 2 3 4 5 6 7 +ywang29-p4d-debug-worker-0:2470947:2470947 [0] NCCL INFO Channel 19/24 : 0 1 2 3 4 5 6 7 +ywang29-p4d-debug-worker-0:2470947:2470947 [0] NCCL INFO Channel 20/24 : 0 1 2 3 4 5 6 7 +ywang29-p4d-debug-worker-0:2470947:2470947 [0] NCCL INFO Channel 21/24 : 0 1 2 3 4 5 6 7 +ywang29-p4d-debug-worker-0:2470947:2470947 [0] NCCL INFO Channel 22/24 : 0 1 2 3 4 5 6 7 +ywang29-p4d-debug-worker-0:2470947:2470947 [0] NCCL INFO Channel 23/24 : 0 1 2 3 4 5 6 7 +ywang29-p4d-debug-worker-0:2470947:2470947 [0] NCCL INFO Trees [0] 1/-1/-1->0->-1 [1] 1/-1/-1->0->-1 [2] 1/-1/-1->0->-1 [3] 1/-1/-1->0->-1 [4] 1/-1/-1->0->-1 [5] 1/-1/-1->0->-1 [6] 1/-1/-1->0->-1 [7] 1/-1/-1->0->-1 [8] 1/-1/-1->0->-1 [9] 1/-1/-1->0->-1 [10] 1/-1/-1->0->-1 [11] 1/-1/-1->0->-1 [12] 1/-1/-1->0->-1 [13] 1/-1/-1->0->-1 [14] 1/-1/-1->0->-1 [15] 1/-1/-1->0->-1 [16] 1/-1/-1->0->-1 [17] 1/-1/-1->0->-1 [18] 1/-1/-1->0->-1 [19] 1/-1/-1->0->-1 [20] 1/-1/-1->0->-1 [21] 1/-1/-1->0->-1 [22] 1/-1/-1->0->-1 [23] 1/-1/-1->0->-1 +ywang29-p4d-debug-worker-0:2470947:2470947 [0] NCCL INFO P2P Chunksize set to 524288 +ywang29-p4d-debug-worker-0:2470948:2470948 [1] NCCL INFO PROFILER/Plugin: Could not find: libnccl-profiler.so +ywang29-p4d-debug-worker-0:2470948:2470948 [1] NCCL INFO Check P2P Type isAllDirectP2p 1 directMode 0 isAllCudaP2p 1 +ywang29-p4d-debug-worker-0:2470948:2471411 [0] NCCL INFO [Proxy Service UDS] Device 1 CPU core 49 +ywang29-p4d-debug-worker-0:2470948:2471410 [0] NCCL INFO [Proxy Service] Device 1 CPU core 23 +ywang29-p4d-debug-worker-0:2470953:2470953 [5] NCCL INFO PROFILER/Plugin: Could not find: libnccl-profiler.so +ywang29-p4d-debug-worker-0:2470953:2470953 [5] NCCL INFO Check P2P Type isAllDirectP2p 1 directMode 0 isAllCudaP2p 1 +ywang29-p4d-debug-worker-0:2470953:2471412 [0] NCCL INFO [Proxy Service] Device 5 CPU core 80 +ywang29-p4d-debug-worker-0:2470953:2471413 [0] NCCL INFO [Proxy Service UDS] Device 5 CPU core 86 +ywang29-p4d-debug-worker-0:2470947:2470947 [0] NCCL INFO PROFILER/Plugin: Could not find: libnccl-profiler.so +ywang29-p4d-debug-worker-0:2470947:2470947 [0] NCCL INFO Check P2P Type isAllDirectP2p 1 directMode 0 isAllCudaP2p 1 +ywang29-p4d-debug-worker-0:2470947:2471414 [0] NCCL INFO [Proxy Service] Device 0 CPU core 15 +ywang29-p4d-debug-worker-0:2470947:2471415 [0] NCCL INFO [Proxy Service UDS] Device 0 CPU core 65 +ywang29-p4d-debug-worker-0:2470949:2470949 [2] NCCL INFO PROFILER/Plugin: Could not find: libnccl-profiler.so +ywang29-p4d-debug-worker-0:2470949:2470949 [2] NCCL INFO Check P2P Type isAllDirectP2p 1 directMode 0 isAllCudaP2p 1 +ywang29-p4d-debug-worker-0:2470949:2471416 [0] NCCL INFO [Proxy Service] Device 2 CPU core 9 +ywang29-p4d-debug-worker-0:2470949:2471417 [0] NCCL INFO [Proxy Service UDS] Device 2 CPU core 9 +ywang29-p4d-debug-worker-0:2470952:2470952 [4] NCCL INFO PROFILER/Plugin: Could not find: libnccl-profiler.so +ywang29-p4d-debug-worker-0:2470952:2470952 [4] NCCL INFO Check P2P Type isAllDirectP2p 1 directMode 0 isAllCudaP2p 1 +ywang29-p4d-debug-worker-0:2470952:2471418 [0] NCCL INFO [Proxy Service] Device 4 CPU core 38 +ywang29-p4d-debug-worker-0:2470955:2470955 [7] NCCL INFO PROFILER/Plugin: Could not find: libnccl-profiler.so +ywang29-p4d-debug-worker-0:2470952:2471419 [0] NCCL INFO [Proxy Service UDS] Device 4 CPU core 39 +ywang29-p4d-debug-worker-0:2470955:2470955 [7] NCCL INFO Check P2P Type isAllDirectP2p 1 directMode 0 isAllCudaP2p 1 +ywang29-p4d-debug-worker-0:2470955:2471421 [0] NCCL INFO [Proxy Service UDS] Device 7 CPU core 34 +ywang29-p4d-debug-worker-0:2470955:2471420 [0] NCCL INFO [Proxy Service] Device 7 CPU core 82 +ywang29-p4d-debug-worker-0:2470954:2470954 [6] NCCL INFO PROFILER/Plugin: Could not find: libnccl-profiler.so +ywang29-p4d-debug-worker-0:2470954:2470954 [6] NCCL INFO Check P2P Type isAllDirectP2p 1 directMode 0 isAllCudaP2p 1 +ywang29-p4d-debug-worker-0:2470954:2471422 [0] NCCL INFO [Proxy Service] Device 6 CPU core 31 +ywang29-p4d-debug-worker-0:2470954:2471423 [0] NCCL INFO [Proxy Service UDS] Device 6 CPU core 31 +ywang29-p4d-debug-worker-0:2470951:2470951 [3] NCCL INFO PROFILER/Plugin: Could not find: libnccl-profiler.so +ywang29-p4d-debug-worker-0:2470951:2470951 [3] NCCL INFO Check P2P Type isAllDirectP2p 1 directMode 0 isAllCudaP2p 1 +ywang29-p4d-debug-worker-0:2470951:2471424 [0] NCCL INFO [Proxy Service] Device 3 CPU core 4 +ywang29-p4d-debug-worker-0:2470951:2471425 [0] NCCL INFO [Proxy Service UDS] Device 3 CPU core 3 +ywang29-p4d-debug-worker-0:2470947:2470947 [0] NCCL INFO TUNER/Plugin: Could not find: libnccl-tuner.so +ywang29-p4d-debug-worker-0:2470947:2470947 [0] NCCL INFO TUNER/Plugin: Using nccl_ofi_tuner (v3) +ywang29-p4d-debug-worker-0:2470947:2470947 [0] NCCL INFO NET/OFI NCCL_OFI_TUNER is not available for platform : p4de.24xlarge, Fall back to NCCL's tuner +ywang29-p4d-debug-worker-0:2470947:2470947 [0] NCCL INFO threadThresholds 8/8/64 | 64/8/64 | 512 | 512 +ywang29-p4d-debug-worker-0:2470947:2470947 [0] NCCL INFO 24 coll channels, 24 collnet channels, 0 nvls channels, 32 p2p channels, 32 p2p channels per peer +ywang29-p4d-debug-worker-0:2470947:2470947 [0] NCCL INFO CC Off, workFifoBytes 1048576 +ywang29-p4d-debug-worker-0:2470954:2470954 [6] NCCL INFO TUNER/Plugin: Could not find: libnccl-tuner.so +ywang29-p4d-debug-worker-0:2470954:2470954 [6] NCCL INFO TUNER/Plugin: Using nccl_ofi_tuner (v3) +ywang29-p4d-debug-worker-0:2470954:2470954 [6] NCCL INFO NET/OFI NCCL_OFI_TUNER is not available for platform : p4de.24xlarge, Fall back to NCCL's tuner +ywang29-p4d-debug-worker-0:2470954:2470954 [6] NCCL INFO threadThresholds 8/8/64 | 64/8/64 | 512 | 512 +ywang29-p4d-debug-worker-0:2470954:2470954 [6] NCCL INFO 24 coll channels, 24 collnet channels, 0 nvls channels, 32 p2p channels, 32 p2p channels per peer +ywang29-p4d-debug-worker-0:2470955:2470955 [7] NCCL INFO TUNER/Plugin: Could not find: libnccl-tuner.so +ywang29-p4d-debug-worker-0:2470955:2470955 [7] NCCL INFO TUNER/Plugin: Using nccl_ofi_tuner (v3) +ywang29-p4d-debug-worker-0:2470955:2470955 [7] NCCL INFO NET/OFI NCCL_OFI_TUNER is not available for platform : p4de.24xlarge, Fall back to NCCL's tuner +ywang29-p4d-debug-worker-0:2470955:2470955 [7] NCCL INFO threadThresholds 8/8/64 | 64/8/64 | 512 | 512 +ywang29-p4d-debug-worker-0:2470955:2470955 [7] NCCL INFO 24 coll channels, 24 collnet channels, 0 nvls channels, 32 p2p channels, 32 p2p channels per peer +ywang29-p4d-debug-worker-0:2470949:2470949 [2] NCCL INFO TUNER/Plugin: Could not find: libnccl-tuner.so +ywang29-p4d-debug-worker-0:2470949:2470949 [2] NCCL INFO TUNER/Plugin: Using nccl_ofi_tuner (v3) +ywang29-p4d-debug-worker-0:2470949:2470949 [2] NCCL INFO NET/OFI NCCL_OFI_TUNER is not available for platform : p4de.24xlarge, Fall back to NCCL's tuner +ywang29-p4d-debug-worker-0:2470949:2470949 [2] NCCL INFO threadThresholds 8/8/64 | 64/8/64 | 512 | 512 +ywang29-p4d-debug-worker-0:2470949:2470949 [2] NCCL INFO 24 coll channels, 24 collnet channels, 0 nvls channels, 32 p2p channels, 32 p2p channels per peer +ywang29-p4d-debug-worker-0:2470953:2470953 [5] NCCL INFO TUNER/Plugin: Could not find: libnccl-tuner.so +ywang29-p4d-debug-worker-0:2470953:2470953 [5] NCCL INFO TUNER/Plugin: Using nccl_ofi_tuner (v3) +ywang29-p4d-debug-worker-0:2470953:2470953 [5] NCCL INFO NET/OFI NCCL_OFI_TUNER is not available for platform : p4de.24xlarge, Fall back to NCCL's tuner +ywang29-p4d-debug-worker-0:2470953:2470953 [5] NCCL INFO threadThresholds 8/8/64 | 64/8/64 | 512 | 512 +ywang29-p4d-debug-worker-0:2470953:2470953 [5] NCCL INFO 24 coll channels, 24 collnet channels, 0 nvls channels, 32 p2p channels, 32 p2p channels per peer +ywang29-p4d-debug-worker-0:2470948:2470948 [1] NCCL INFO TUNER/Plugin: Could not find: libnccl-tuner.so +ywang29-p4d-debug-worker-0:2470948:2470948 [1] NCCL INFO TUNER/Plugin: Using nccl_ofi_tuner (v3) +ywang29-p4d-debug-worker-0:2470948:2470948 [1] NCCL INFO NET/OFI NCCL_OFI_TUNER is not available for platform : p4de.24xlarge, Fall back to NCCL's tuner +ywang29-p4d-debug-worker-0:2470948:2470948 [1] NCCL INFO threadThresholds 8/8/64 | 64/8/64 | 512 | 512 +ywang29-p4d-debug-worker-0:2470948:2470948 [1] NCCL INFO 24 coll channels, 24 collnet channels, 0 nvls channels, 32 p2p channels, 32 p2p channels per peer +ywang29-p4d-debug-worker-0:2470951:2470951 [3] NCCL INFO TUNER/Plugin: Could not find: libnccl-tuner.so +ywang29-p4d-debug-worker-0:2470952:2470952 [4] NCCL INFO TUNER/Plugin: Could not find: libnccl-tuner.so +ywang29-p4d-debug-worker-0:2470951:2470951 [3] NCCL INFO TUNER/Plugin: Using nccl_ofi_tuner (v3) +ywang29-p4d-debug-worker-0:2470952:2470952 [4] NCCL INFO TUNER/Plugin: Using nccl_ofi_tuner (v3) +ywang29-p4d-debug-worker-0:2470951:2470951 [3] NCCL INFO NET/OFI NCCL_OFI_TUNER is not available for platform : p4de.24xlarge, Fall back to NCCL's tuner +ywang29-p4d-debug-worker-0:2470951:2470951 [3] NCCL INFO threadThresholds 8/8/64 | 64/8/64 | 512 | 512 +ywang29-p4d-debug-worker-0:2470951:2470951 [3] NCCL INFO 24 coll channels, 24 collnet channels, 0 nvls channels, 32 p2p channels, 32 p2p channels per peer +ywang29-p4d-debug-worker-0:2470952:2470952 [4] NCCL INFO NET/OFI NCCL_OFI_TUNER is not available for platform : p4de.24xlarge, Fall back to NCCL's tuner +ywang29-p4d-debug-worker-0:2470952:2470952 [4] NCCL INFO threadThresholds 8/8/64 | 64/8/64 | 512 | 512 +ywang29-p4d-debug-worker-0:2470952:2470952 [4] NCCL INFO 24 coll channels, 24 collnet channels, 0 nvls channels, 32 p2p channels, 32 p2p channels per peer +ywang29-p4d-debug-worker-0:2470949:2470949 [2] NCCL INFO ncclCommInitRankConfig comm 0x5602809d0750 rank 2 nranks 8 cudaDev 2 nvmlDev 2 busId 201c0 commId 0xbdc52b61bae6206d - Init COMPLETE +ywang29-p4d-debug-worker-0:2470954:2470954 [6] NCCL INFO ncclCommInitRankConfig comm 0x56071a5c6320 rank 6 nranks 8 cudaDev 6 nvmlDev 6 busId a01c0 commId 0xbdc52b61bae6206d - Init COMPLETE +ywang29-p4d-debug-worker-0:2470952:2470952 [4] NCCL INFO ncclCommInitRankConfig comm 0x5647078795e0 rank 4 nranks 8 cudaDev 4 nvmlDev 4 busId 901c0 commId 0xbdc52b61bae6206d - Init COMPLETE +ywang29-p4d-debug-worker-0:2470947:2470947 [0] NCCL INFO ncclCommInitRankConfig comm 0x55ffbe0f7c80 rank 0 nranks 8 cudaDev 0 nvmlDev 0 busId 101c0 commId 0xbdc52b61bae6206d - Init COMPLETE +ywang29-p4d-debug-worker-0:2470949:2470949 [2] NCCL INFO Init timings - ncclCommInitRankConfig: rank 2 nranks 8 total 0.52 (kernels 0.17, alloc 0.22, bootstrap 0.00, allgathers 0.00, topo 0.04, graphs 0.00, connections 0.06, rest 0.04) +ywang29-p4d-debug-worker-0:2470952:2470952 [4] NCCL INFO Init timings - ncclCommInitRankConfig: rank 4 nranks 8 total 0.53 (kernels 0.18, alloc 0.22, bootstrap 0.00, allgathers 0.00, topo 0.03, graphs 0.00, connections 0.06, rest 0.04) +ywang29-p4d-debug-worker-0:2470954:2470954 [6] NCCL INFO Init timings - ncclCommInitRankConfig: rank 6 nranks 8 total 0.53 (kernels 0.18, alloc 0.22, bootstrap 0.01, allgathers 0.00, topo 0.04, graphs 0.00, connections 0.05, rest 0.04) +ywang29-p4d-debug-worker-0:2470947:2470947 [0] NCCL INFO Init timings - ncclCommInitRankConfig: rank 0 nranks 8 total 0.53 (kernels 0.18, alloc 0.22, bootstrap 0.01, allgathers 0.00, topo 0.04, graphs 0.00, connections 0.06, rest 0.03) +ywang29-p4d-debug-worker-0:2470948:2470948 [1] NCCL INFO ncclCommInitRankConfig comm 0x558cf7486a10 rank 1 nranks 8 cudaDev 1 nvmlDev 1 busId 101d0 commId 0xbdc52b61bae6206d - Init COMPLETE +ywang29-p4d-debug-worker-0:2470955:2470955 [7] NCCL INFO ncclCommInitRankConfig comm 0x55e36d363080 rank 7 nranks 8 cudaDev 7 nvmlDev 7 busId a01d0 commId 0xbdc52b61bae6206d - Init COMPLETE +ywang29-p4d-debug-worker-0:2470953:2470953 [5] NCCL INFO ncclCommInitRankConfig comm 0x55c0201d0a60 rank 5 nranks 8 cudaDev 5 nvmlDev 5 busId 901d0 commId 0xbdc52b61bae6206d - Init COMPLETE +ywang29-p4d-debug-worker-0:2470948:2470948 [1] NCCL INFO Init timings - ncclCommInitRankConfig: rank 1 nranks 8 total 0.53 (kernels 0.18, alloc 0.22, bootstrap 0.01, allgathers 0.00, topo 0.04, graphs 0.00, connections 0.08, rest 0.02) +ywang29-p4d-debug-worker-0:2470955:2470955 [7] NCCL INFO Init timings - ncclCommInitRankConfig: rank 7 nranks 8 total 0.53 (kernels 0.18, alloc 0.20, bootstrap 0.03, allgathers 0.00, topo 0.04, graphs 0.00, connections 0.06, rest 0.04) +ywang29-p4d-debug-worker-0:2470953:2470953 [5] NCCL INFO Init timings - ncclCommInitRankConfig: rank 5 nranks 8 total 0.53 (kernels 0.18, alloc 0.22, bootstrap 0.00, allgathers 0.00, topo 0.04, graphs 0.00, connections 0.08, rest 0.02) +ywang29-p4d-debug-worker-0:2470951:2470951 [3] NCCL INFO ncclCommInitRankConfig comm 0x5612d38dcbf0 rank 3 nranks 8 cudaDev 3 nvmlDev 3 busId 201d0 commId 0xbdc52b61bae6206d - Init COMPLETE +ywang29-p4d-debug-worker-0:2470951:2470951 [3] NCCL INFO Init timings - ncclCommInitRankConfig: rank 3 nranks 8 total 0.53 (kernels 0.18, alloc 0.22, bootstrap 0.00, allgathers 0.00, topo 0.03, graphs 0.00, connections 0.06, rest 0.04) +[INFO|2026-04-29 15:42:35] llamafactory.hparams.parser:505 >> Process rank: 5, world size: 8, device: cuda:5, distributed training: True, compute dtype: torch.bfloat16 +[INFO|2026-04-29 15:42:35] llamafactory.hparams.parser:505 >> Process rank: 4, world size: 8, device: cuda:4, distributed training: True, compute dtype: torch.bfloat16 +[INFO|2026-04-29 15:42:35] llamafactory.hparams.parser:505 >> Process rank: 6, world size: 8, device: cuda:6, distributed training: True, compute dtype: torch.bfloat16 +[INFO|2026-04-29 15:42:35] llamafactory.hparams.parser:505 >> Process rank: 7, world size: 8, device: cuda:7, distributed training: True, compute dtype: torch.bfloat16 +[INFO|2026-04-29 15:42:35] llamafactory.hparams.parser:505 >> Process rank: 1, world size: 8, device: cuda:1, distributed training: True, compute dtype: torch.bfloat16 +[INFO|2026-04-29 15:42:35] llamafactory.hparams.parser:505 >> Process rank: 2, world size: 8, device: cuda:2, distributed training: True, compute dtype: torch.bfloat16 +[INFO|2026-04-29 15:42:35] llamafactory.hparams.parser:505 >> Process rank: 3, world size: 8, device: cuda:3, distributed training: True, compute dtype: torch.bfloat16 +[INFO|2026-04-29 15:42:35] llamafactory.hparams.parser:505 >> Process rank: 0, world size: 8, device: cuda:0, distributed training: True, compute dtype: torch.bfloat16 +[INFO|configuration_utils.py:670] 2026-04-29 15:42:35,891 >> loading configuration file config.json from cache at /root/.cache/huggingface/hub/models--Qwen--Qwen3-1.7B/snapshots/70d244cc86ccca08cf5af4e1e306ecf908b1ad5e/config.json +[INFO|configuration_utils.py:742] 2026-04-29 15:42:35,894 >> Model config Qwen3Config { + "architectures": [ + "Qwen3ForCausalLM" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "bos_token_id": 151643, + "dtype": "bfloat16", + "eos_token_id": 151645, + "head_dim": 128, + "hidden_act": "silu", + "hidden_size": 2048, + "initializer_range": 0.02, + "intermediate_size": 6144, + "layer_types": [ + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention" + ], + "masked_layers": null, + "max_position_embeddings": 40960, + "max_window_layers": 28, + "model_type": "qwen3", + "num_attention_heads": 16, + "num_hidden_layers": 28, + "num_key_value_heads": 8, + "pad_token_id": null, + "rms_norm_eps": 1e-06, + "rope_parameters": { + "rope_theta": 1000000, + "rope_type": "default" + }, + "sliding_window": null, + "sparsity_attn": null, + "sparsity_mlp": null, + "subnet_mode": null, + "subnet_type": null, + "threshold_attn": null, + "threshold_mlp": null, + "tie_word_embeddings": true, + "transformers_version": "5.2.0", + "use_cache": true, + "use_sliding_window": false, + "vocab_size": 151936 +} + +[INFO|configuration_utils.py:670] 2026-04-29 15:42:39,796 >> loading configuration file config.json from cache at /root/.cache/huggingface/hub/models--Qwen--Qwen3-1.7B/snapshots/70d244cc86ccca08cf5af4e1e306ecf908b1ad5e/config.json +[INFO|configuration_utils.py:742] 2026-04-29 15:42:39,797 >> Model config Qwen3Config { + "architectures": [ + "Qwen3ForCausalLM" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "bos_token_id": 151643, + "dtype": "bfloat16", + "eos_token_id": 151645, + "head_dim": 128, + "hidden_act": "silu", + "hidden_size": 2048, + "initializer_range": 0.02, + "intermediate_size": 6144, + "layer_types": [ + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention" + ], + "masked_layers": null, + "max_position_embeddings": 40960, + "max_window_layers": 28, + "model_type": "qwen3", + "num_attention_heads": 16, + "num_hidden_layers": 28, + "num_key_value_heads": 8, + "pad_token_id": null, + "rms_norm_eps": 1e-06, + "rope_parameters": { + "rope_theta": 1000000, + "rope_type": "default" + }, + "sliding_window": null, + "sparsity_attn": null, + "sparsity_mlp": null, + "subnet_mode": null, + "subnet_type": null, + "threshold_attn": null, + "threshold_mlp": null, + "tie_word_embeddings": true, + "transformers_version": "5.2.0", + "use_cache": true, + "use_sliding_window": false, + "vocab_size": 151936 +} + +[INFO|configuration_utils.py:670] 2026-04-29 15:42:39,906 >> loading configuration file config.json from cache at /root/.cache/huggingface/hub/models--Qwen--Qwen3-1.7B/snapshots/70d244cc86ccca08cf5af4e1e306ecf908b1ad5e/config.json +[INFO|configuration_utils.py:742] 2026-04-29 15:42:39,907 >> Model config Qwen3Config { + "architectures": [ + "Qwen3ForCausalLM" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "bos_token_id": 151643, + "dtype": "bfloat16", + "eos_token_id": 151645, + "head_dim": 128, + "hidden_act": "silu", + "hidden_size": 2048, + "initializer_range": 0.02, + "intermediate_size": 6144, + "layer_types": [ + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention" + ], + "masked_layers": null, + "max_position_embeddings": 40960, + "max_window_layers": 28, + "model_type": "qwen3", + "num_attention_heads": 16, + "num_hidden_layers": 28, + "num_key_value_heads": 8, + "pad_token_id": null, + "rms_norm_eps": 1e-06, + "rope_parameters": { + "rope_theta": 1000000, + "rope_type": "default" + }, + "sliding_window": null, + "sparsity_attn": null, + "sparsity_mlp": null, + "subnet_mode": null, + "subnet_type": null, + "threshold_attn": null, + "threshold_mlp": null, + "tie_word_embeddings": true, + "transformers_version": "5.2.0", + "use_cache": true, + "use_sliding_window": false, + "vocab_size": 151936 +} + +ywang29-p4d-debug-worker-0:2470952:2470952 [4] NCCL INFO Channel 00/0 : 4[4] -> 5[5] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470952:2470952 [4] NCCL INFO Channel 01/0 : 4[4] -> 5[5] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470952:2470952 [4] NCCL INFO Channel 02/0 : 4[4] -> 5[5] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470952:2470952 [4] NCCL INFO Channel 03/0 : 4[4] -> 5[5] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470952:2470952 [4] NCCL INFO Channel 04/0 : 4[4] -> 5[5] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470952:2470952 [4] NCCL INFO Channel 05/0 : 4[4] -> 5[5] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470952:2470952 [4] NCCL INFO Channel 06/0 : 4[4] -> 5[5] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470952:2470952 [4] NCCL INFO Channel 07/0 : 4[4] -> 5[5] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470952:2470952 [4] NCCL INFO Channel 08/0 : 4[4] -> 5[5] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470952:2470952 [4] NCCL INFO Channel 09/0 : 4[4] -> 5[5] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470952:2470952 [4] NCCL INFO Channel 10/0 : 4[4] -> 5[5] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470952:2470952 [4] NCCL INFO Channel 11/0 : 4[4] -> 5[5] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470952:2470952 [4] NCCL INFO Channel 12/0 : 4[4] -> 5[5] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470952:2470952 [4] NCCL INFO Channel 13/0 : 4[4] -> 5[5] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470952:2470952 [4] NCCL INFO Channel 14/0 : 4[4] -> 5[5] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470952:2470952 [4] NCCL INFO Channel 15/0 : 4[4] -> 5[5] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470952:2470952 [4] NCCL INFO Channel 16/0 : 4[4] -> 5[5] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470952:2470952 [4] NCCL INFO Channel 17/0 : 4[4] -> 5[5] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470952:2470952 [4] NCCL INFO Channel 18/0 : 4[4] -> 5[5] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470952:2470952 [4] NCCL INFO Channel 19/0 : 4[4] -> 5[5] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470952:2470952 [4] NCCL INFO Channel 20/0 : 4[4] -> 5[5] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470952:2470952 [4] NCCL INFO Channel 21/0 : 4[4] -> 5[5] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470952:2470952 [4] NCCL INFO Channel 22/0 : 4[4] -> 5[5] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470952:2470952 [4] NCCL INFO Channel 23/0 : 4[4] -> 5[5] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470954:2470954 [6] NCCL INFO Channel 00/0 : 6[6] -> 7[7] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470954:2470954 [6] NCCL INFO Channel 01/0 : 6[6] -> 7[7] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470954:2470954 [6] NCCL INFO Channel 02/0 : 6[6] -> 7[7] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470954:2470954 [6] NCCL INFO Channel 03/0 : 6[6] -> 7[7] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470954:2470954 [6] NCCL INFO Channel 04/0 : 6[6] -> 7[7] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470954:2470954 [6] NCCL INFO Channel 05/0 : 6[6] -> 7[7] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470954:2470954 [6] NCCL INFO Channel 06/0 : 6[6] -> 7[7] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470954:2470954 [6] NCCL INFO Channel 07/0 : 6[6] -> 7[7] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470954:2470954 [6] NCCL INFO Channel 08/0 : 6[6] -> 7[7] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470954:2470954 [6] NCCL INFO Channel 09/0 : 6[6] -> 7[7] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470954:2470954 [6] NCCL INFO Channel 10/0 : 6[6] -> 7[7] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470954:2470954 [6] NCCL INFO Channel 11/0 : 6[6] -> 7[7] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470954:2470954 [6] NCCL INFO Channel 12/0 : 6[6] -> 7[7] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470954:2470954 [6] NCCL INFO Channel 13/0 : 6[6] -> 7[7] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470954:2470954 [6] NCCL INFO Channel 14/0 : 6[6] -> 7[7] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470954:2470954 [6] NCCL INFO Channel 15/0 : 6[6] -> 7[7] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470954:2470954 [6] NCCL INFO Channel 16/0 : 6[6] -> 7[7] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470954:2470954 [6] NCCL INFO Channel 17/0 : 6[6] -> 7[7] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470954:2470954 [6] NCCL INFO Channel 18/0 : 6[6] -> 7[7] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470954:2470954 [6] NCCL INFO Channel 19/0 : 6[6] -> 7[7] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470954:2470954 [6] NCCL INFO Channel 20/0 : 6[6] -> 7[7] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470954:2470954 [6] NCCL INFO Channel 21/0 : 6[6] -> 7[7] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470954:2470954 [6] NCCL INFO Channel 22/0 : 6[6] -> 7[7] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470954:2470954 [6] NCCL INFO Channel 23/0 : 6[6] -> 7[7] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470948:2470948 [1] NCCL INFO Channel 00/0 : 1[1] -> 2[2] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470948:2470948 [1] NCCL INFO Channel 01/0 : 1[1] -> 2[2] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470948:2470948 [1] NCCL INFO Channel 02/0 : 1[1] -> 2[2] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470948:2470948 [1] NCCL INFO Channel 03/0 : 1[1] -> 2[2] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470948:2470948 [1] NCCL INFO Channel 04/0 : 1[1] -> 2[2] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470948:2470948 [1] NCCL INFO Channel 05/0 : 1[1] -> 2[2] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470948:2470948 [1] NCCL INFO Channel 06/0 : 1[1] -> 2[2] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470948:2470948 [1] NCCL INFO Channel 07/0 : 1[1] -> 2[2] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470948:2470948 [1] NCCL INFO Channel 08/0 : 1[1] -> 2[2] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470948:2470948 [1] NCCL INFO Channel 09/0 : 1[1] -> 2[2] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470948:2470948 [1] NCCL INFO Channel 10/0 : 1[1] -> 2[2] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470948:2470948 [1] NCCL INFO Channel 11/0 : 1[1] -> 2[2] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470948:2470948 [1] NCCL INFO Channel 12/0 : 1[1] -> 2[2] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470948:2470948 [1] NCCL INFO Channel 13/0 : 1[1] -> 2[2] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470948:2470948 [1] NCCL INFO Channel 14/0 : 1[1] -> 2[2] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470948:2470948 [1] NCCL INFO Channel 15/0 : 1[1] -> 2[2] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470948:2470948 [1] NCCL INFO Channel 16/0 : 1[1] -> 2[2] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470948:2470948 [1] NCCL INFO Channel 17/0 : 1[1] -> 2[2] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470948:2470948 [1] NCCL INFO Channel 18/0 : 1[1] -> 2[2] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470948:2470948 [1] NCCL INFO Channel 19/0 : 1[1] -> 2[2] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470948:2470948 [1] NCCL INFO Channel 20/0 : 1[1] -> 2[2] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470948:2470948 [1] NCCL INFO Channel 21/0 : 1[1] -> 2[2] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470948:2470948 [1] NCCL INFO Channel 22/0 : 1[1] -> 2[2] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470948:2470948 [1] NCCL INFO Channel 23/0 : 1[1] -> 2[2] via P2P/CUMEM/read +[INFO|2026-04-29 15:42:41] llamafactory.data.loader:144 >> Loading dataset KKHYA/metamath_converted... +ywang29-p4d-debug-worker-0:2470951:2470951 [3] NCCL INFO Channel 00/0 : 3[3] -> 4[4] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470951:2470951 [3] NCCL INFO Channel 01/0 : 3[3] -> 4[4] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470951:2470951 [3] NCCL INFO Channel 02/0 : 3[3] -> 4[4] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470951:2470951 [3] NCCL INFO Channel 03/0 : 3[3] -> 4[4] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470951:2470951 [3] NCCL INFO Channel 04/0 : 3[3] -> 4[4] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470951:2470951 [3] NCCL INFO Channel 05/0 : 3[3] -> 4[4] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470951:2470951 [3] NCCL INFO Channel 06/0 : 3[3] -> 4[4] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470951:2470951 [3] NCCL INFO Channel 07/0 : 3[3] -> 4[4] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470951:2470951 [3] NCCL INFO Channel 08/0 : 3[3] -> 4[4] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470951:2470951 [3] NCCL INFO Channel 09/0 : 3[3] -> 4[4] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470951:2470951 [3] NCCL INFO Channel 10/0 : 3[3] -> 4[4] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470951:2470951 [3] NCCL INFO Channel 11/0 : 3[3] -> 4[4] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470951:2470951 [3] NCCL INFO Channel 12/0 : 3[3] -> 4[4] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470951:2470951 [3] NCCL INFO Channel 13/0 : 3[3] -> 4[4] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470951:2470951 [3] NCCL INFO Channel 14/0 : 3[3] -> 4[4] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470951:2470951 [3] NCCL INFO Channel 15/0 : 3[3] -> 4[4] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470951:2470951 [3] NCCL INFO Channel 16/0 : 3[3] -> 4[4] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470951:2470951 [3] NCCL INFO Channel 17/0 : 3[3] -> 4[4] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470951:2470951 [3] NCCL INFO Channel 18/0 : 3[3] -> 4[4] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470951:2470951 [3] NCCL INFO Channel 19/0 : 3[3] -> 4[4] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470951:2470951 [3] NCCL INFO Channel 20/0 : 3[3] -> 4[4] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470951:2470951 [3] NCCL INFO Channel 21/0 : 3[3] -> 4[4] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470951:2470951 [3] NCCL INFO Channel 22/0 : 3[3] -> 4[4] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470951:2470951 [3] NCCL INFO Channel 23/0 : 3[3] -> 4[4] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470955:2470955 [7] NCCL INFO Channel 00/0 : 7[7] -> 0[0] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470955:2470955 [7] NCCL INFO Channel 01/0 : 7[7] -> 0[0] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470955:2470955 [7] NCCL INFO Channel 02/0 : 7[7] -> 0[0] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470955:2470955 [7] NCCL INFO Channel 03/0 : 7[7] -> 0[0] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470955:2470955 [7] NCCL INFO Channel 04/0 : 7[7] -> 0[0] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470955:2470955 [7] NCCL INFO Channel 05/0 : 7[7] -> 0[0] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470955:2470955 [7] NCCL INFO Channel 06/0 : 7[7] -> 0[0] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470955:2470955 [7] NCCL INFO Channel 07/0 : 7[7] -> 0[0] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470955:2470955 [7] NCCL INFO Channel 08/0 : 7[7] -> 0[0] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470955:2470955 [7] NCCL INFO Channel 09/0 : 7[7] -> 0[0] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470955:2470955 [7] NCCL INFO Channel 10/0 : 7[7] -> 0[0] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470955:2470955 [7] NCCL INFO Channel 11/0 : 7[7] -> 0[0] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470955:2470955 [7] NCCL INFO Channel 12/0 : 7[7] -> 0[0] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470955:2470955 [7] NCCL INFO Channel 13/0 : 7[7] -> 0[0] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470955:2470955 [7] NCCL INFO Channel 14/0 : 7[7] -> 0[0] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470953:2470953 [5] NCCL INFO Channel 00/0 : 5[5] -> 6[6] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470955:2470955 [7] NCCL INFO Channel 15/0 : 7[7] -> 0[0] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470953:2470953 [5] NCCL INFO Channel 01/0 : 5[5] -> 6[6] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470955:2470955 [7] NCCL INFO Channel 16/0 : 7[7] -> 0[0] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470953:2470953 [5] NCCL INFO Channel 02/0 : 5[5] -> 6[6] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470955:2470955 [7] NCCL INFO Channel 17/0 : 7[7] -> 0[0] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470955:2470955 [7] NCCL INFO Channel 18/0 : 7[7] -> 0[0] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470953:2470953 [5] NCCL INFO Channel 03/0 : 5[5] -> 6[6] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470953:2470953 [5] NCCL INFO Channel 04/0 : 5[5] -> 6[6] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470955:2470955 [7] NCCL INFO Channel 19/0 : 7[7] -> 0[0] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470955:2470955 [7] NCCL INFO Channel 20/0 : 7[7] -> 0[0] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470953:2470953 [5] NCCL INFO Channel 05/0 : 5[5] -> 6[6] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470955:2470955 [7] NCCL INFO Channel 21/0 : 7[7] -> 0[0] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470953:2470953 [5] NCCL INFO Channel 06/0 : 5[5] -> 6[6] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470955:2470955 [7] NCCL INFO Channel 22/0 : 7[7] -> 0[0] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470953:2470953 [5] NCCL INFO Channel 07/0 : 5[5] -> 6[6] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470955:2470955 [7] NCCL INFO Channel 23/0 : 7[7] -> 0[0] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470953:2470953 [5] NCCL INFO Channel 08/0 : 5[5] -> 6[6] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470953:2470953 [5] NCCL INFO Channel 09/0 : 5[5] -> 6[6] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470953:2470953 [5] NCCL INFO Channel 10/0 : 5[5] -> 6[6] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470953:2470953 [5] NCCL INFO Channel 11/0 : 5[5] -> 6[6] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470953:2470953 [5] NCCL INFO Channel 12/0 : 5[5] -> 6[6] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470953:2470953 [5] NCCL INFO Channel 13/0 : 5[5] -> 6[6] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470953:2470953 [5] NCCL INFO Channel 14/0 : 5[5] -> 6[6] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470953:2470953 [5] NCCL INFO Channel 15/0 : 5[5] -> 6[6] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470953:2470953 [5] NCCL INFO Channel 16/0 : 5[5] -> 6[6] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470953:2470953 [5] NCCL INFO Channel 17/0 : 5[5] -> 6[6] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470953:2470953 [5] NCCL INFO Channel 18/0 : 5[5] -> 6[6] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470953:2470953 [5] NCCL INFO Channel 19/0 : 5[5] -> 6[6] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470953:2470953 [5] NCCL INFO Channel 20/0 : 5[5] -> 6[6] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470953:2470953 [5] NCCL INFO Channel 21/0 : 5[5] -> 6[6] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470953:2470953 [5] NCCL INFO Channel 22/0 : 5[5] -> 6[6] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470953:2470953 [5] NCCL INFO Channel 23/0 : 5[5] -> 6[6] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470949:2470949 [2] NCCL INFO Channel 00/0 : 2[2] -> 3[3] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470949:2470949 [2] NCCL INFO Channel 01/0 : 2[2] -> 3[3] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470949:2470949 [2] NCCL INFO Channel 02/0 : 2[2] -> 3[3] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470949:2470949 [2] NCCL INFO Channel 03/0 : 2[2] -> 3[3] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470949:2470949 [2] NCCL INFO Channel 04/0 : 2[2] -> 3[3] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470949:2470949 [2] NCCL INFO Channel 05/0 : 2[2] -> 3[3] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470949:2470949 [2] NCCL INFO Channel 06/0 : 2[2] -> 3[3] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470949:2470949 [2] NCCL INFO Channel 07/0 : 2[2] -> 3[3] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470949:2470949 [2] NCCL INFO Channel 08/0 : 2[2] -> 3[3] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470949:2470949 [2] NCCL INFO Channel 09/0 : 2[2] -> 3[3] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470949:2470949 [2] NCCL INFO Channel 10/0 : 2[2] -> 3[3] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470949:2470949 [2] NCCL INFO Channel 11/0 : 2[2] -> 3[3] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470949:2470949 [2] NCCL INFO Channel 12/0 : 2[2] -> 3[3] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470949:2470949 [2] NCCL INFO Channel 13/0 : 2[2] -> 3[3] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470949:2470949 [2] NCCL INFO Channel 14/0 : 2[2] -> 3[3] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470949:2470949 [2] NCCL INFO Channel 15/0 : 2[2] -> 3[3] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470949:2470949 [2] NCCL INFO Channel 16/0 : 2[2] -> 3[3] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470949:2470949 [2] NCCL INFO Channel 17/0 : 2[2] -> 3[3] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470949:2470949 [2] NCCL INFO Channel 18/0 : 2[2] -> 3[3] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470949:2470949 [2] NCCL INFO Channel 19/0 : 2[2] -> 3[3] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470949:2470949 [2] NCCL INFO Channel 20/0 : 2[2] -> 3[3] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470949:2470949 [2] NCCL INFO Channel 21/0 : 2[2] -> 3[3] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470949:2470949 [2] NCCL INFO Channel 22/0 : 2[2] -> 3[3] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470949:2470949 [2] NCCL INFO Channel 23/0 : 2[2] -> 3[3] via P2P/CUMEM/read +Repo card metadata block was not found. Setting CardData to empty. +Setting num_proc from 16 to 2 for the train split as it only contains 2 shards. + Generating train split: 0%| | 0/394996 [00:00> Sampled 10000 examples from dataset KKHYA/metamath_converted. + Converting format of dataset (num_proc=16): 0%| | 0/10000 [00:00> Loading dataset KKHYA/numinamath_tir_converted... +Repo card metadata block was not found. Setting CardData to empty. +Setting num_proc from 16 back to 1 for the train split to disable multiprocessing as it only contains one shard. + Generating train split: 0%| | 0/72441 [00:00> Sampled 10000 examples from dataset KKHYA/numinamath_tir_converted. + Converting format of dataset (num_proc=16): 0%| | 0/10000 [00:00> Loading dataset KKHYA/numinamath_cot_converted... +Repo card metadata block was not found. Setting CardData to empty. +Setting num_proc from 16 to 5 for the train split as it only contains 5 shards. + Generating train split: 0%| | 0/859493 [00:00> Sampled 10000 examples from dataset KKHYA/numinamath_cot_converted. + Converting format of dataset (num_proc=16): 0%| | 0/10000 [00:00> Loading dataset allenai/tulu-3-sft-personas-math... +Setting num_proc from 16 to 2 for the train split as it only contains 2 shards. + Generating train split: 0%| | 0/149960 [00:00> Sampled 10000 examples from dataset allenai/tulu-3-sft-personas-math. + Converting format of dataset (num_proc=16): 0%| | 0/10000 [00:00> Loading dataset allenai/tulu-3-sft-personas-math-grade... +Setting num_proc from 16 back to 1 for the train split to disable multiprocessing as it only contains one shard. + Generating train split: 0%| | 0/49980 [00:00> Sampled 10000 examples from dataset allenai/tulu-3-sft-personas-math-grade. + Converting format of dataset (num_proc=16): 0%| | 0/10000 [00:00> Loading dataset allenai/tulu-3-sft-personas-algebra... +Setting num_proc from 16 back to 1 for the train split to disable multiprocessing as it only contains one shard. + Generating train split: 0%| | 0/20000 [00:00> Sampled 10000 examples from dataset allenai/tulu-3-sft-personas-algebra. + Converting format of dataset (num_proc=16): 0%| | 0/10000 [00:00 1[1] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470947:2470947 [0] NCCL INFO Channel 01/0 : 0[0] -> 1[1] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470947:2470947 [0] NCCL INFO Channel 02/0 : 0[0] -> 1[1] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470947:2470947 [0] NCCL INFO Channel 03/0 : 0[0] -> 1[1] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470947:2470947 [0] NCCL INFO Channel 04/0 : 0[0] -> 1[1] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470947:2470947 [0] NCCL INFO Channel 05/0 : 0[0] -> 1[1] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470947:2470947 [0] NCCL INFO Channel 06/0 : 0[0] -> 1[1] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470947:2470947 [0] NCCL INFO Channel 07/0 : 0[0] -> 1[1] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470947:2470947 [0] NCCL INFO Channel 08/0 : 0[0] -> 1[1] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470947:2470947 [0] NCCL INFO Channel 09/0 : 0[0] -> 1[1] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470947:2470947 [0] NCCL INFO Channel 10/0 : 0[0] -> 1[1] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470947:2470947 [0] NCCL INFO Channel 11/0 : 0[0] -> 1[1] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470947:2470947 [0] NCCL INFO Channel 12/0 : 0[0] -> 1[1] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470947:2470947 [0] NCCL INFO Channel 13/0 : 0[0] -> 1[1] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470947:2470947 [0] NCCL INFO Channel 14/0 : 0[0] -> 1[1] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470947:2470947 [0] NCCL INFO Channel 15/0 : 0[0] -> 1[1] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470947:2470947 [0] NCCL INFO Channel 16/0 : 0[0] -> 1[1] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470947:2470947 [0] NCCL INFO Channel 17/0 : 0[0] -> 1[1] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470947:2470947 [0] NCCL INFO Channel 18/0 : 0[0] -> 1[1] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470947:2470947 [0] NCCL INFO Channel 19/0 : 0[0] -> 1[1] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470947:2470947 [0] NCCL INFO Channel 20/0 : 0[0] -> 1[1] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470947:2470947 [0] NCCL INFO Channel 21/0 : 0[0] -> 1[1] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470947:2470947 [0] NCCL INFO Channel 22/0 : 0[0] -> 1[1] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470947:2470947 [0] NCCL INFO Channel 23/0 : 0[0] -> 1[1] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470955:2470955 [7] NCCL INFO Connected all rings, use ring PXN 0 GDR 1 +ywang29-p4d-debug-worker-0:2470947:2470947 [0] NCCL INFO Connected all rings, use ring PXN 0 GDR 1 +ywang29-p4d-debug-worker-0:2470954:2470954 [6] NCCL INFO Connected all rings, use ring PXN 0 GDR 1 +ywang29-p4d-debug-worker-0:2470948:2470948 [1] NCCL INFO Connected all rings, use ring PXN 0 GDR 1 +ywang29-p4d-debug-worker-0:2470949:2470949 [2] NCCL INFO Connected all rings, use ring PXN 0 GDR 1 +ywang29-p4d-debug-worker-0:2470953:2470953 [5] NCCL INFO Connected all rings, use ring PXN 0 GDR 1 +ywang29-p4d-debug-worker-0:2470951:2470951 [3] NCCL INFO Connected all rings, use ring PXN 0 GDR 1 +ywang29-p4d-debug-worker-0:2470952:2470952 [4] NCCL INFO Connected all rings, use ring PXN 0 GDR 1 +Repo card metadata block was not found. Setting CardData to empty. +Repo card metadata block was not found. Setting CardData to empty. +Repo card metadata block was not found. Setting CardData to empty. +Repo card metadata block was not found. Setting CardData to empty. +Repo card metadata block was not found. Setting CardData to empty. +Repo card metadata block was not found. Setting CardData to empty. +Repo card metadata block was not found. Setting CardData to empty. + Running tokenizer on dataset (num_proc=16): 0%| | 0/60000 [00:00user +Jean has three times as much money as Jane. They have a combined total of $76. How much money does Jean have?<|im_end|> +<|im_start|>assistant +Let's assume Jane has x dollars. +Then Jean has 3x dollars. +The total amount of money they have is x + 3x = 4x +We know that the total amount of money they have is $76, so 4x = $76 +Dividing both sides of the equation by 4, we get x = $19 +Since Jean has three times as much money as Jane, Jean has 3 * $19 = $57 +Therefore, Jean has $57 +#### 57 +The answer is: 57<|im_end|> + +label_ids: +[-100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, 10061, 594, 9658, 21475, 702, 856, 11192, 624, 12209, 19685, 702, 220, 18, 87, 11192, 624, 785, 2790, 3311, 315, 3220, 807, 614, 374, 856, 488, 220, 18, 87, 284, 220, 19, 87, 198, 1654, 1414, 429, 279, 2790, 3311, 315, 3220, 807, 614, 374, 400, 22, 21, 11, 773, 220, 19, 87, 284, 400, 22, 21, 198, 12509, 6577, 2176, 11067, 315, 279, 23606, 553, 220, 19, 11, 582, 633, 856, 284, 400, 16, 24, 198, 12549, 19685, 702, 2326, 3039, 438, 1753, 3220, 438, 21475, 11, 19685, 702, 220, 18, 353, 400, 16, 24, 284, 400, 20, 22, 198, 54815, 11, 19685, 702, 400, 20, 22, 198, 820, 220, 20, 22, 198, 785, 4226, 374, 25, 220, 20, 22, 151645, 198] +labels: +Let's assume Jane has x dollars. +Then Jean has 3x dollars. +The total amount of money they have is x + 3x = 4x +We know that the total amount of money they have is $76, so 4x = $76 +Dividing both sides of the equation by 4, we get x = $19 +Since Jean has three times as much money as Jane, Jean has 3 * $19 = $57 +Therefore, Jean has $57 +#### 57 +The answer is: 57<|im_end|> + +[INFO|configuration_utils.py:670] 2026-04-29 15:44:15,949 >> loading configuration file config.json from cache at /root/.cache/huggingface/hub/models--Qwen--Qwen3-1.7B/snapshots/70d244cc86ccca08cf5af4e1e306ecf908b1ad5e/config.json +[INFO|configuration_utils.py:742] 2026-04-29 15:44:15,949 >> Model config Qwen3Config { + "architectures": [ + "Qwen3ForCausalLM" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "bos_token_id": 151643, + "dtype": "bfloat16", + "eos_token_id": 151645, + "head_dim": 128, + "hidden_act": "silu", + "hidden_size": 2048, + "initializer_range": 0.02, + "intermediate_size": 6144, + "layer_types": [ + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention" + ], + "masked_layers": null, + "max_position_embeddings": 40960, + "max_window_layers": 28, + "model_type": "qwen3", + "num_attention_heads": 16, + "num_hidden_layers": 28, + "num_key_value_heads": 8, + "pad_token_id": null, + "rms_norm_eps": 1e-06, + "rope_parameters": { + "rope_theta": 1000000, + "rope_type": "default" + }, + "sliding_window": null, + "sparsity_attn": null, + "sparsity_mlp": null, + "subnet_mode": null, + "subnet_type": null, + "threshold_attn": null, + "threshold_mlp": null, + "tie_word_embeddings": true, + "transformers_version": "5.2.0", + "use_cache": true, + "use_sliding_window": false, + "vocab_size": 151936 +} + +[INFO|2026-04-29 15:44:15] llamafactory.model.model_utils.kv_cache:144 >> KV cache is disabled during training. +[INFO|modeling_utils.py:710] 2026-04-29 15:44:17,465 >> loading weights file model.safetensors from cache at /root/.cache/huggingface/hub/models--Qwen--Qwen3-1.7B/snapshots/70d244cc86ccca08cf5af4e1e306ecf908b1ad5e/model.safetensors.index.json + Fetching 2 files: 0%| | 0/2 [00:00> Will use dtype=torch.bfloat16 as defined in model's config object + Fetching 2 files: 50%|█████ | 1/2 [00:04<00:04, 4.02s/it] Fetching 2 files: 100%|██████████| 2/2 [00:04<00:00, 2.01s/it] +[INFO|configuration_utils.py:1014] 2026-04-29 15:44:22,426 >> Generate config GenerationConfig { + "bos_token_id": 151643, + "eos_token_id": 151645, + "output_attentions": false, + "output_hidden_states": false, + "use_cache": false +} + + Fetching 2 files: 50%|█████ | 1/2 [00:04<00:04, 4.02s/it] Fetching 2 files: 100%|██████████| 2/2 [00:04<00:00, 2.01s/it] + Fetching 2 files: 50%|█████ | 1/2 [00:03<00:03, 3.98s/it] Fetching 2 files: 100%|██████████| 2/2 [00:03<00:00, 1.99s/it] + Fetching 2 files: 50%|█████ | 1/2 [00:03<00:03, 3.98s/it] Fetching 2 files: 100%|██████████| 2/2 [00:03<00:00, 1.99s/it] + Fetching 2 files: 50%|█████ | 1/2 [00:04<00:04, 4.07s/it] Fetching 2 files: 100%|██████████| 2/2 [00:04<00:00, 2.04s/it] + Fetching 2 files: 50%|█████ | 1/2 [00:04<00:04, 4.17s/it] Fetching 2 files: 100%|██████████| 2/2 [00:04<00:00, 2.08s/it] + Fetching 2 files: 50%|█████ | 1/2 [00:04<00:04, 4.12s/it] Fetching 2 files: 100%|██████████| 2/2 [00:04<00:00, 2.06s/it] + Loading weights: 0%| | 0/311 [00:00> The tied weights mapping and config for this model specifies to tie model.embed_tokens.weight to lm_head.weight, but both are present in the checkpoints, so we will NOT tie them. You should update the config with `tie_word_embeddings=False` to silence this warning + Loading weights: 86%|████████▋ | 269/311 [00:01<00:00, 255.68it/s, Materializing param=model.layers.24.mlp.gate_proj.weight] Loading weights: 88%|████████▊ | 275/311 [00:01<00:00, 261.82it/s, Materializing param=model.layers.24.self_attn.q_norm.weight] Loading weights: 87%|████████▋ | 270/311 [00:01<00:00, 255.68it/s, Materializing param=model.layers.24.mlp.up_proj.weight] Loading weights: 84%|████████▍ | 262/311 [00:01<00:00, 184.15it/s, Materializing param=model.layers.23.self_attn.k_proj.weight] Loading weights: 82%|████████▏ | 254/311 [00:01<00:00, 245.34it/s, Materializing param=model.layers.22.self_attn.q_norm.weight] Loading weights: 88%|████████▊ | 275/311 [00:01<00:00, 261.82it/s, Materializing param=model.layers.24.self_attn.q_norm.weight] Loading weights: 87%|████████▋ | 270/311 [00:01<00:00, 255.68it/s, Materializing param=model.layers.24.mlp.up_proj.weight] Loading weights: 86%|████████▌ | 266/311 [00:01<00:00, 247.65it/s, Materializing param=model.layers.23.self_attn.q_proj.weight] Loading weights: 82%|████████▏ | 254/311 [00:01<00:00, 245.34it/s, Materializing param=model.layers.22.self_attn.q_proj.weight] Loading weights: 55%|█████▍ | 171/311 [00:01<00:00, 172.13it/s, Materializing param=model.layers.15.mlp.gate_proj.weight] Loading weights: 89%|████████▊ | 276/311 [00:01<00:00, 261.82it/s, Materializing param=model.layers.24.self_attn.q_proj.weight] Loading weights: 87%|████████▋ | 271/311 [00:01<00:00, 255.68it/s, Materializing param=model.layers.24.post_attention_layernorm.weight] Loading weights: 89%|████████▊ | 276/311 [00:01<00:00, 261.82it/s, Materializing param=model.layers.24.self_attn.q_proj.weight] Loading weights: 82%|████████▏ | 254/311 [00:01<00:00, 245.34it/s, Materializing param=model.layers.22.self_attn.q_proj.weight] Loading weights: 87%|████████▋ | 271/311 [00:01<00:00, 255.68it/s, Materializing param=model.layers.24.post_attention_layernorm.weight] Loading weights: 86%|████████▌ | 266/311 [00:01<00:00, 247.65it/s, Materializing param=model.layers.23.self_attn.v_proj.weight] Loading weights: 55%|█████▍ | 171/311 [00:01<00:00, 172.13it/s, Materializing param=model.layers.15.mlp.up_proj.weight] Loading weights: 85%|████████▍ | 263/311 [00:01<00:00, 250.91it/s, Materializing param=model.layers.23.self_attn.k_proj.weight] Loading weights: 86%|████████▌ | 266/311 [00:01<00:00, 247.65it/s, Materializing param=model.layers.23.self_attn.v_proj.weight] Loading weights: 89%|████████▉ | 277/311 [00:01<00:00, 261.82it/s, Materializing param=model.layers.24.self_attn.v_proj.weight] Loading weights: 87%|████████▋ | 272/311 [00:01<00:00, 255.68it/s, Materializing param=model.layers.24.self_attn.k_norm.weight] Loading weights: 55%|█████▍ | 171/311 [00:01<00:00, 172.13it/s, Materializing param=model.layers.15.mlp.up_proj.weight] Loading weights: 82%|████████▏ | 255/311 [00:01<00:00, 245.34it/s, Materializing param=model.layers.22.self_attn.v_proj.weight] Loading weights: 89%|████████▉ | 277/311 [00:01<00:00, 261.82it/s, Materializing param=model.layers.24.self_attn.v_proj.weight] Loading weights: 85%|████████▍ | 263/311 [00:01<00:00, 250.91it/s, Materializing param=model.layers.23.self_attn.o_proj.weight] Loading weights: 87%|████████▋ | 272/311 [00:01<00:00, 255.68it/s, Materializing param=model.layers.24.self_attn.k_norm.weight] Loading weights: 82%|████████▏ | 255/311 [00:01<00:00, 245.34it/s, Materializing param=model.layers.22.self_attn.v_proj.weight] Loading weights: 85%|████████▍ | 263/311 [00:01<00:00, 250.91it/s, Materializing param=model.layers.23.self_attn.o_proj.weight] Loading weights: 89%|████████▉ | 278/311 [00:01<00:00, 261.82it/s, Materializing param=model.layers.25.input_layernorm.weight] Loading weights: 86%|████████▌ | 267/311 [00:01<00:00, 247.65it/s, Materializing param=model.layers.24.input_layernorm.weight] Loading weights: 88%|████████▊ | 273/311 [00:01<00:00, 255.68it/s, Materializing param=model.layers.24.self_attn.k_proj.weight] Loading weights: 82%|████████▏ | 256/311 [00:01<00:00, 245.34it/s, Materializing param=model.layers.23.input_layernorm.weight] Loading weights: 89%|████████▉ | 278/311 [00:01<00:00, 261.82it/s, Materializing param=model.layers.25.input_layernorm.weight] Loading weights: 86%|████████▌ | 267/311 [00:01<00:00, 247.65it/s, Materializing param=model.layers.24.input_layernorm.weight] Loading weights: 88%|████████▊ | 273/311 [00:01<00:00, 255.68it/s, Materializing param=model.layers.24.self_attn.k_proj.weight] Loading weights: 55%|█████▌ | 172/311 [00:01<00:00, 172.13it/s, Materializing param=model.layers.15.post_attention_layernorm.weight] Loading weights: 85%|████████▍ | 264/311 [00:01<00:00, 250.91it/s, Materializing param=model.layers.23.self_attn.q_norm.weight] Loading weights: 82%|████████▏ | 256/311 [00:01<00:00, 245.34it/s, Materializing param=model.layers.23.input_layernorm.weight] Loading weights: 85%|████████▍ | 264/311 [00:01<00:00, 250.91it/s, Materializing param=model.layers.23.self_attn.q_norm.weight] Loading weights: 90%|████████▉ | 279/311 [00:01<00:00, 261.82it/s, Materializing param=model.layers.25.mlp.down_proj.weight] Loading weights: 55%|█████▌ | 172/311 [00:01<00:00, 172.13it/s, Materializing param=model.layers.15.post_attention_layernorm.weight] Loading weights: 86%|████████▌ | 268/311 [00:01<00:00, 247.65it/s, Materializing param=model.layers.24.mlp.down_proj.weight] Loading weights: 88%|████████▊ | 274/311 [00:01<00:00, 255.68it/s, Materializing param=model.layers.24.self_attn.o_proj.weight] Loading weights: 90%|████████▉ | 279/311 [00:01<00:00, 261.82it/s, Materializing param=model.layers.25.mlp.down_proj.weight] Loading weights: 83%|████████▎ | 257/311 [00:01<00:00, 245.34it/s, Materializing param=model.layers.23.mlp.down_proj.weight] Loading weights: 88%|████████▊ | 274/311 [00:01<00:00, 255.68it/s, Materializing param=model.layers.24.self_attn.o_proj.weight] Loading weights: 86%|████████▌ | 268/311 [00:01<00:00, 247.65it/s, Materializing param=model.layers.24.mlp.down_proj.weight] Loading weights: 85%|████████▌ | 265/311 [00:01<00:00, 250.91it/s, Materializing param=model.layers.23.self_attn.q_proj.weight] Loading weights: 83%|████████▎ | 257/311 [00:01<00:00, 245.34it/s, Materializing param=model.layers.23.mlp.down_proj.weight] Loading weights: 56%|█████▌ | 173/311 [00:01<00:00, 172.13it/s, Materializing param=model.layers.15.self_attn.k_norm.weight] Loading weights: 90%|█████████ | 280/311 [00:01<00:00, 261.82it/s, Materializing param=model.layers.25.mlp.gate_proj.weight] Loading weights: 85%|████████▌ | 265/311 [00:01<00:00, 250.91it/s, Materializing param=model.layers.23.self_attn.q_proj.weight] Loading weights: 88%|████████▊ | 275/311 [00:01<00:00, 255.68it/s, Materializing param=model.layers.24.self_attn.q_norm.weight] Loading weights: 86%|████████▋ | 269/311 [00:01<00:00, 247.65it/s, Materializing param=model.layers.24.mlp.gate_proj.weight] Loading weights: 90%|█████████ | 280/311 [00:01<00:00, 261.82it/s, Materializing param=model.layers.25.mlp.gate_proj.weight] Loading weights: 88%|████████▊ | 275/311 [00:01<00:00, 255.68it/s, Materializing param=model.layers.24.self_attn.q_norm.weight] Loading weights: 83%|████████▎ | 258/311 [00:01<00:00, 245.34it/s, Materializing param=model.layers.23.mlp.gate_proj.weight] Loading weights: 86%|████████▋ | 269/311 [00:01<00:00, 247.65it/s, Materializing param=model.layers.24.mlp.gate_proj.weight] Loading weights: 56%|█████▌ | 173/311 [00:01<00:00, 172.13it/s, Materializing param=model.layers.15.self_attn.k_norm.weight] Loading weights: 86%|████████▌ | 266/311 [00:01<00:00, 250.91it/s, Materializing param=model.layers.23.self_attn.v_proj.weight] Loading weights: 83%|████████▎ | 258/311 [00:01<00:00, 245.34it/s, Materializing param=model.layers.23.mlp.gate_proj.weight] Loading weights: 90%|█████████ | 281/311 [00:01<00:00, 261.82it/s, Materializing param=model.layers.25.mlp.up_proj.weight] Loading weights: 89%|████████▊ | 276/311 [00:01<00:00, 255.68it/s, Materializing param=model.layers.24.self_attn.q_proj.weight] Loading weights: 86%|████████▌ | 266/311 [00:01<00:00, 250.91it/s, Materializing param=model.layers.23.self_attn.v_proj.weight] Loading weights: 87%|████████▋ | 270/311 [00:01<00:00, 247.65it/s, Materializing param=model.layers.24.mlp.up_proj.weight] Loading weights: 90%|█████████ | 281/311 [00:01<00:00, 261.82it/s, Materializing param=model.layers.25.mlp.up_proj.weight] Loading weights: 89%|████████▊ | 276/311 [00:01<00:00, 255.68it/s, Materializing param=model.layers.24.self_attn.q_proj.weight] Loading weights: 87%|████████▋ | 270/311 [00:01<00:00, 247.65it/s, Materializing param=model.layers.24.mlp.up_proj.weight] Loading weights: 56%|█████▌ | 174/311 [00:01<00:00, 172.13it/s, Materializing param=model.layers.15.self_attn.k_proj.weight] Loading weights: 83%|████████▎ | 259/311 [00:01<00:00, 245.34it/s, Materializing param=model.layers.23.mlp.up_proj.weight] Loading weights: 86%|████████▌ | 267/311 [00:01<00:00, 250.91it/s, Materializing param=model.layers.24.input_layernorm.weight] Loading weights: 83%|████████▎ | 259/311 [00:01<00:00, 245.34it/s, Materializing param=model.layers.23.mlp.up_proj.weight] Loading weights: 91%|█████████ | 282/311 [00:01<00:00, 261.82it/s, Materializing param=model.layers.25.post_attention_layernorm.weight] Loading weights: 89%|████████▉ | 277/311 [00:01<00:00, 255.68it/s, Materializing param=model.layers.24.self_attn.v_proj.weight] Loading weights: 56%|█████▌ | 174/311 [00:01<00:00, 172.13it/s, Materializing param=model.layers.15.self_attn.k_proj.weight] Loading weights: 86%|████████▌ | 267/311 [00:01<00:00, 250.91it/s, Materializing param=model.layers.24.input_layernorm.weight] Loading weights: 87%|████████▋ | 271/311 [00:01<00:00, 247.65it/s, Materializing param=model.layers.24.post_attention_layernorm.weight] Loading weights: 91%|█████████ | 282/311 [00:01<00:00, 261.82it/s, Materializing param=model.layers.25.post_attention_layernorm.weight] Loading weights: 89%|████████▉ | 277/311 [00:01<00:00, 255.68it/s, Materializing param=model.layers.24.self_attn.v_proj.weight] Loading weights: 87%|████████▋ | 271/311 [00:01<00:00, 247.65it/s, Materializing param=model.layers.24.post_attention_layernorm.weight] Loading weights: 84%|████████▎ | 260/311 [00:01<00:00, 245.34it/s, Materializing param=model.layers.23.post_attention_layernorm.weight] Loading weights: 86%|████████▌ | 268/311 [00:01<00:00, 250.91it/s, Materializing param=model.layers.24.mlp.down_proj.weight] Loading weights: 91%|█████████ | 283/311 [00:01<00:00, 261.82it/s, Materializing param=model.layers.25.self_attn.k_norm.weight] Loading weights: 84%|████████▎ | 260/311 [00:01<00:00, 245.34it/s, Materializing param=model.layers.23.post_attention_layernorm.weight] Loading weights: 89%|████████▉ | 278/311 [00:01<00:00, 255.68it/s, Materializing param=model.layers.25.input_layernorm.weight] Loading weights: 56%|█████▋ | 175/311 [00:01<00:00, 172.13it/s, Materializing param=model.layers.15.self_attn.o_proj.weight] Loading weights: 87%|████████▋ | 272/311 [00:01<00:00, 247.65it/s, Materializing param=model.layers.24.self_attn.k_norm.weight] Loading weights: 86%|████████▌ | 268/311 [00:01<00:00, 250.91it/s, Materializing param=model.layers.24.mlp.down_proj.weight] Loading weights: 91%|█████████ | 283/311 [00:01<00:00, 261.82it/s, Materializing param=model.layers.25.self_attn.k_norm.weight] Loading weights: 89%|████████▉ | 278/311 [00:01<00:00, 255.68it/s, Materializing param=model.layers.25.input_layernorm.weight] Loading weights: 87%|████████▋ | 272/311 [00:01<00:00, 247.65it/s, Materializing param=model.layers.24.self_attn.k_norm.weight] Loading weights: 56%|█████▋ | 175/311 [00:01<00:00, 172.13it/s, Materializing param=model.layers.15.self_attn.o_proj.weight] Loading weights: 84%|████████▍ | 261/311 [00:01<00:00, 245.34it/s, Materializing param=model.layers.23.self_attn.k_norm.weight] Loading weights: 91%|█████████▏| 284/311 [00:01<00:00, 261.82it/s, Materializing param=model.layers.25.self_attn.k_proj.weight] Loading weights: 86%|████████▋ | 269/311 [00:01<00:00, 250.91it/s, Materializing param=model.layers.24.mlp.gate_proj.weight] Loading weights: 90%|████████▉ | 279/311 [00:01<00:00, 255.68it/s, Materializing param=model.layers.25.mlp.down_proj.weight] Loading weights: 84%|████████▍ | 261/311 [00:01<00:00, 245.34it/s, Materializing param=model.layers.23.self_attn.k_norm.weight] Loading weights: 88%|████████▊ | 273/311 [00:01<00:00, 247.65it/s, Materializing param=model.layers.24.self_attn.k_proj.weight] Loading weights: 91%|█████████▏| 284/311 [00:01<00:00, 261.82it/s, Materializing param=model.layers.25.self_attn.k_proj.weight] Loading weights: 90%|████████▉ | 279/311 [00:01<00:00, 255.68it/s, Materializing param=model.layers.25.mlp.down_proj.weight] Loading weights: 86%|████████▋ | 269/311 [00:01<00:00, 250.91it/s, Materializing param=model.layers.24.mlp.gate_proj.weight] Loading weights: 57%|█████▋ | 176/311 [00:01<00:00, 172.13it/s, Materializing param=model.layers.15.self_attn.q_norm.weight] Loading weights: 88%|████████▊ | 273/311 [00:01<00:00, 247.65it/s, Materializing param=model.layers.24.self_attn.k_proj.weight] Loading weights: 84%|████████▍ | 262/311 [00:01<00:00, 245.34it/s, Materializing param=model.layers.23.self_attn.k_proj.weight] Loading weights: 92%|█████████▏| 285/311 [00:01<00:00, 261.82it/s, Materializing param=model.layers.25.self_attn.o_proj.weight] Loading weights: 90%|█████████ | 280/311 [00:01<00:00, 255.68it/s, Materializing param=model.layers.25.mlp.gate_proj.weight] Loading weights: 87%|████████▋ | 270/311 [00:01<00:00, 250.91it/s, Materializing param=model.layers.24.mlp.up_proj.weight] Loading weights: 57%|█████▋ | 176/311 [00:01<00:00, 172.13it/s, Materializing param=model.layers.15.self_attn.q_norm.weight] Loading weights: 84%|████████▍ | 262/311 [00:01<00:00, 245.34it/s, Materializing param=model.layers.23.self_attn.k_proj.weight] Loading weights: 92%|█████████▏| 285/311 [00:01<00:00, 261.82it/s, Materializing param=model.layers.25.self_attn.o_proj.weight] Loading weights: 90%|█████████ | 280/311 [00:01<00:00, 255.68it/s, Materializing param=model.layers.25.mlp.gate_proj.weight] Loading weights: 87%|████████▋ | 270/311 [00:01<00:00, 250.91it/s, Materializing param=model.layers.24.mlp.up_proj.weight] Loading weights: 88%|████████▊ | 274/311 [00:01<00:00, 247.65it/s, Materializing param=model.layers.24.self_attn.o_proj.weight] Loading weights: 88%|████████▊ | 274/311 [00:01<00:00, 247.65it/s, Materializing param=model.layers.24.self_attn.o_proj.weight] Loading weights: 85%|████████▍ | 263/311 [00:01<00:00, 245.34it/s, Materializing param=model.layers.23.self_attn.o_proj.weight] Loading weights: 90%|█████████ | 281/311 [00:01<00:00, 255.68it/s, Materializing param=model.layers.25.mlp.up_proj.weight] Loading weights: 92%|█████████▏| 286/311 [00:01<00:00, 261.82it/s, Materializing param=model.layers.25.self_attn.q_norm.weight] Loading weights: 57%|█████▋ | 177/311 [00:01<00:00, 172.13it/s, Materializing param=model.layers.15.self_attn.q_proj.weight] Loading weights: 87%|████████▋ | 271/311 [00:01<00:00, 250.91it/s, Materializing param=model.layers.24.post_attention_layernorm.weight] Loading weights: 85%|████████▍ | 263/311 [00:01<00:00, 245.34it/s, Materializing param=model.layers.23.self_attn.o_proj.weight] Loading weights: 90%|█████████ | 281/311 [00:01<00:00, 255.68it/s, Materializing param=model.layers.25.mlp.up_proj.weight] Loading weights: 92%|█████████▏| 286/311 [00:01<00:00, 261.82it/s, Materializing param=model.layers.25.self_attn.q_norm.weight] Loading weights: 88%|████████▊ | 275/311 [00:01<00:00, 247.65it/s, Materializing param=model.layers.24.self_attn.q_norm.weight] Loading weights: 87%|████████▋ | 271/311 [00:01<00:00, 250.91it/s, Materializing param=model.layers.24.post_attention_layernorm.weight] Loading weights: 57%|█████▋ | 177/311 [00:01<00:00, 172.13it/s, Materializing param=model.layers.15.self_attn.q_proj.weight] Loading weights: 88%|████████▊ | 275/311 [00:01<00:00, 247.65it/s, Materializing param=model.layers.24.self_attn.q_norm.weight] Loading weights: 85%|████████▍ | 264/311 [00:01<00:00, 245.34it/s, Materializing param=model.layers.23.self_attn.q_norm.weight] Loading weights: 91%|█████████ | 282/311 [00:01<00:00, 255.68it/s, Materializing param=model.layers.25.post_attention_layernorm.weight] Loading weights: 92%|█████████▏| 287/311 [00:01<00:00, 261.82it/s, Materializing param=model.layers.25.self_attn.q_proj.weight] Loading weights: 87%|████████▋ | 272/311 [00:01<00:00, 250.91it/s, Materializing param=model.layers.24.self_attn.k_norm.weight] Loading weights: 85%|████████▍ | 264/311 [00:01<00:00, 245.34it/s, Materializing param=model.layers.23.self_attn.q_norm.weight] Loading weights: 91%|█████████ | 282/311 [00:01<00:00, 255.68it/s, Materializing param=model.layers.25.post_attention_layernorm.weight] Loading weights: 92%|█████████▏| 287/311 [00:01<00:00, 261.82it/s, Materializing param=model.layers.25.self_attn.q_proj.weight] Loading weights: 89%|████████▊ | 276/311 [00:01<00:00, 247.65it/s, Materializing param=model.layers.24.self_attn.q_proj.weight] Loading weights: 87%|████████▋ | 272/311 [00:01<00:00, 250.91it/s, Materializing param=model.layers.24.self_attn.k_norm.weight] Loading weights: 57%|█████▋ | 178/311 [00:01<00:00, 172.13it/s, Materializing param=model.layers.15.self_attn.v_proj.weight] Loading weights: 89%|████████▊ | 276/311 [00:01<00:00, 247.65it/s, Materializing param=model.layers.24.self_attn.q_proj.weight] Loading weights: 91%|█████████ | 283/311 [00:01<00:00, 255.68it/s, Materializing param=model.layers.25.self_attn.k_norm.weight] Loading weights: 93%|█████████▎| 288/311 [00:01<00:00, 261.82it/s, Materializing param=model.layers.25.self_attn.v_proj.weight] Loading weights: 85%|████████▌ | 265/311 [00:01<00:00, 245.34it/s, Materializing param=model.layers.23.self_attn.q_proj.weight] Loading weights: 57%|█████▋ | 178/311 [00:01<00:00, 172.13it/s, Materializing param=model.layers.15.self_attn.v_proj.weight] Loading weights: 93%|█████████▎| 288/311 [00:01<00:00, 261.82it/s, Materializing param=model.layers.25.self_attn.v_proj.weight] Loading weights: 88%|████████▊ | 273/311 [00:01<00:00, 250.91it/s, Materializing param=model.layers.24.self_attn.k_proj.weight] Loading weights: 91%|█████████ | 283/311 [00:01<00:00, 255.68it/s, Materializing param=model.layers.25.self_attn.k_norm.weight] Loading weights: 85%|████████▌ | 265/311 [00:01<00:00, 245.34it/s, Materializing param=model.layers.23.self_attn.q_proj.weight] Loading weights: 89%|████████▉ | 277/311 [00:01<00:00, 247.65it/s, Materializing param=model.layers.24.self_attn.v_proj.weight] Loading weights: 88%|████████▊ | 273/311 [00:01<00:00, 250.91it/s, Materializing param=model.layers.24.self_attn.k_proj.weight] Loading weights: 89%|████████▉ | 277/311 [00:01<00:00, 247.65it/s, Materializing param=model.layers.24.self_attn.v_proj.weight] Loading weights: 93%|█████████▎| 289/311 [00:01<00:00, 261.82it/s, Materializing param=model.layers.26.input_layernorm.weight] Loading weights: 91%|█████████▏| 284/311 [00:01<00:00, 255.68it/s, Materializing param=model.layers.25.self_attn.k_proj.weight] Loading weights: 86%|████████▌ | 266/311 [00:01<00:00, 245.34it/s, Materializing param=model.layers.23.self_attn.v_proj.weight] Loading weights: 58%|█████▊ | 179/311 [00:01<00:00, 172.13it/s, Materializing param=model.layers.16.input_layernorm.weight] Loading weights: 93%|█████████▎| 289/311 [00:01<00:00, 261.82it/s, Materializing param=model.layers.26.input_layernorm.weight] Loading weights: 91%|█████████▏| 284/311 [00:01<00:00, 255.68it/s, Materializing param=model.layers.25.self_attn.k_proj.weight] Loading weights: 86%|████████▌ | 266/311 [00:01<00:00, 245.34it/s, Materializing param=model.layers.23.self_attn.v_proj.weight] Loading weights: 88%|████████▊ | 274/311 [00:01<00:00, 250.91it/s, Materializing param=model.layers.24.self_attn.o_proj.weight] Loading weights: 89%|████████▉ | 278/311 [00:01<00:00, 247.65it/s, Materializing param=model.layers.25.input_layernorm.weight] Loading weights: 58%|█████▊ | 179/311 [00:01<00:00, 172.13it/s, Materializing param=model.layers.16.input_layernorm.weight] Loading weights: 88%|████████▊ | 274/311 [00:01<00:00, 250.91it/s, Materializing param=model.layers.24.self_attn.o_proj.weight] Loading weights: 89%|████████▉ | 278/311 [00:01<00:00, 247.65it/s, Materializing param=model.layers.25.input_layernorm.weight] Loading weights: 93%|█████████▎| 290/311 [00:01<00:00, 261.82it/s, Materializing param=model.layers.26.mlp.down_proj.weight] Loading weights: 92%|█████████▏| 285/311 [00:01<00:00, 255.68it/s, Materializing param=model.layers.25.self_attn.o_proj.weight] Loading weights: 86%|████████▌ | 267/311 [00:01<00:00, 245.34it/s, Materializing param=model.layers.24.input_layernorm.weight] Loading weights: 93%|█████████▎| 290/311 [00:01<00:00, 261.82it/s, Materializing param=model.layers.26.mlp.down_proj.weight] Loading weights: 92%|█████████▏| 285/311 [00:01<00:00, 255.68it/s, Materializing param=model.layers.25.self_attn.o_proj.weight] Loading weights: 86%|████████▌ | 267/311 [00:01<00:00, 245.34it/s, Materializing param=model.layers.24.input_layernorm.weight] Loading weights: 88%|████████▊ | 275/311 [00:01<00:00, 250.91it/s, Materializing param=model.layers.24.self_attn.q_norm.weight] Loading weights: 90%|████████▉ | 279/311 [00:01<00:00, 247.65it/s, Materializing param=model.layers.25.mlp.down_proj.weight] Loading weights: 58%|█████▊ | 180/311 [00:01<00:00, 172.13it/s, Materializing param=model.layers.16.mlp.down_proj.weight] Loading weights: 88%|████████▊ | 275/311 [00:01<00:00, 250.91it/s, Materializing param=model.layers.24.self_attn.q_norm.weight] Loading weights: 94%|█████████▎| 291/311 [00:01<00:00, 261.82it/s, Materializing param=model.layers.26.mlp.gate_proj.weight] Loading weights: 90%|████████▉ | 279/311 [00:01<00:00, 247.65it/s, Materializing param=model.layers.25.mlp.down_proj.weight] Loading weights: 92%|█████████▏| 286/311 [00:01<00:00, 255.68it/s, Materializing param=model.layers.25.self_attn.q_norm.weight] Loading weights: 86%|████████▌ | 268/311 [00:01<00:00, 245.34it/s, Materializing param=model.layers.24.mlp.down_proj.weight] Loading weights: 58%|█████▊ | 180/311 [00:01<00:00, 172.13it/s, Materializing param=model.layers.16.mlp.down_proj.weight] Loading weights: 94%|█████████▎| 291/311 [00:01<00:00, 261.82it/s, Materializing param=model.layers.26.mlp.gate_proj.weight] Loading weights: 92%|█████████▏| 286/311 [00:01<00:00, 255.68it/s, Materializing param=model.layers.25.self_attn.q_norm.weight] Loading weights: 86%|████████▌ | 268/311 [00:01<00:00, 245.34it/s, Materializing param=model.layers.24.mlp.down_proj.weight] Loading weights: 89%|████████▊ | 276/311 [00:01<00:00, 250.91it/s, Materializing param=model.layers.24.self_attn.q_proj.weight] Loading weights: 90%|█████████ | 280/311 [00:01<00:00, 247.65it/s, Materializing param=model.layers.25.mlp.gate_proj.weight] Loading weights: 89%|████████▊ | 276/311 [00:01<00:00, 250.91it/s, Materializing param=model.layers.24.self_attn.q_proj.weight] Loading weights: 92%|█████████▏| 287/311 [00:01<00:00, 255.68it/s, Materializing param=model.layers.25.self_attn.q_proj.weight] Loading weights: 94%|█████████▍| 292/311 [00:01<00:00, 261.82it/s, Materializing param=model.layers.26.mlp.up_proj.weight] Loading weights: 90%|█████████ | 280/311 [00:01<00:00, 247.65it/s, Materializing param=model.layers.25.mlp.gate_proj.weight] Loading weights: 86%|████████▋ | 269/311 [00:01<00:00, 245.34it/s, Materializing param=model.layers.24.mlp.gate_proj.weight] Loading weights: 58%|█████▊ | 181/311 [00:01<00:00, 172.13it/s, Materializing param=model.layers.16.mlp.gate_proj.weight] Loading weights: 94%|█████████▍| 292/311 [00:01<00:00, 261.82it/s, Materializing param=model.layers.26.mlp.up_proj.weight] Loading weights: 92%|█████████▏| 287/311 [00:01<00:00, 255.68it/s, Materializing param=model.layers.25.self_attn.q_proj.weight] Loading weights: 86%|████████▋ | 269/311 [00:01<00:00, 245.34it/s, Materializing param=model.layers.24.mlp.gate_proj.weight] Loading weights: 89%|████████▉ | 277/311 [00:01<00:00, 250.91it/s, Materializing param=model.layers.24.self_attn.v_proj.weight] Loading weights: 90%|█████████ | 281/311 [00:01<00:00, 247.65it/s, Materializing param=model.layers.25.mlp.up_proj.weight] Loading weights: 58%|█████▊ | 181/311 [00:01<00:00, 172.13it/s, Materializing param=model.layers.16.mlp.gate_proj.weight] Loading weights: 89%|████████▉ | 277/311 [00:01<00:00, 250.91it/s, Materializing param=model.layers.24.self_attn.v_proj.weight] Loading weights: 94%|█████████▍| 293/311 [00:01<00:00, 261.82it/s, Materializing param=model.layers.26.post_attention_layernorm.weight] Loading weights: 90%|█████████ | 281/311 [00:01<00:00, 247.65it/s, Materializing param=model.layers.25.mlp.up_proj.weight] Loading weights: 93%|█████████▎| 288/311 [00:01<00:00, 255.68it/s, Materializing param=model.layers.25.self_attn.v_proj.weight] Loading weights: 87%|████████▋ | 270/311 [00:01<00:00, 245.34it/s, Materializing param=model.layers.24.mlp.up_proj.weight] Loading weights: 94%|█████████▍| 293/311 [00:01<00:00, 261.82it/s, Materializing param=model.layers.26.post_attention_layernorm.weight] Loading weights: 93%|█████████▎| 288/311 [00:01<00:00, 255.68it/s, Materializing param=model.layers.25.self_attn.v_proj.weight] Loading weights: 87%|████████▋ | 270/311 [00:01<00:00, 245.34it/s, Materializing param=model.layers.24.mlp.up_proj.weight] Loading weights: 89%|████████▉ | 278/311 [00:01<00:00, 250.91it/s, Materializing param=model.layers.25.input_layernorm.weight] Loading weights: 91%|█████████ | 282/311 [00:01<00:00, 247.65it/s, Materializing param=model.layers.25.post_attention_layernorm.weight] Loading weights: 59%|█████▊ | 182/311 [00:01<00:00, 172.13it/s, Materializing param=model.layers.16.mlp.up_proj.weight] Loading weights: 89%|████████▉ | 278/311 [00:01<00:00, 250.91it/s, Materializing param=model.layers.25.input_layernorm.weight] Loading weights: 95%|█████████▍| 294/311 [00:01<00:00, 261.82it/s, Materializing param=model.layers.26.self_attn.k_norm.weight] Loading weights: 91%|█████████ | 282/311 [00:01<00:00, 247.65it/s, Materializing param=model.layers.25.post_attention_layernorm.weight] Loading weights: 93%|█████████▎| 289/311 [00:01<00:00, 255.68it/s, Materializing param=model.layers.26.input_layernorm.weight] Loading weights: 87%|████████▋ | 271/311 [00:01<00:00, 245.34it/s, Materializing param=model.layers.24.post_attention_layernorm.weight] Loading weights: 95%|█████████▍| 294/311 [00:01<00:00, 261.82it/s, Materializing param=model.layers.26.self_attn.k_norm.weight] Loading weights: 59%|█████▊ | 182/311 [00:01<00:00, 172.13it/s, Materializing param=model.layers.16.mlp.up_proj.weight] Loading weights: 93%|█████████▎| 289/311 [00:01<00:00, 255.68it/s, Materializing param=model.layers.26.input_layernorm.weight] Loading weights: 87%|████████▋ | 271/311 [00:01<00:00, 245.34it/s, Materializing param=model.layers.24.post_attention_layernorm.weight] Loading weights: 90%|████████▉ | 279/311 [00:01<00:00, 250.91it/s, Materializing param=model.layers.25.mlp.down_proj.weight] Loading weights: 91%|█████████ | 283/311 [00:01<00:00, 247.65it/s, Materializing param=model.layers.25.self_attn.k_norm.weight] Loading weights: 90%|████████▉ | 279/311 [00:01<00:00, 250.91it/s, Materializing param=model.layers.25.mlp.down_proj.weight] Loading weights: 95%|█████████▍| 295/311 [00:01<00:00, 261.82it/s, Materializing param=model.layers.26.self_attn.k_proj.weight] Loading weights: 91%|█████████ | 283/311 [00:01<00:00, 247.65it/s, Materializing param=model.layers.25.self_attn.k_norm.weight] Loading weights: 93%|█████████▎| 290/311 [00:01<00:00, 255.68it/s, Materializing param=model.layers.26.mlp.down_proj.weight] Loading weights: 87%|████████▋ | 272/311 [00:01<00:00, 245.34it/s, Materializing param=model.layers.24.self_attn.k_norm.weight] Loading weights: 95%|█████████▍| 295/311 [00:01<00:00, 261.82it/s, Materializing param=model.layers.26.self_attn.k_proj.weight] Loading weights: 93%|█████████▎| 290/311 [00:01<00:00, 255.68it/s, Materializing param=model.layers.26.mlp.down_proj.weight] Loading weights: 59%|█████▉ | 183/311 [00:01<00:00, 172.13it/s, Materializing param=model.layers.16.post_attention_layernorm.weight] Loading weights: 87%|████████▋ | 272/311 [00:01<00:00, 245.34it/s, Materializing param=model.layers.24.self_attn.k_norm.weight] Loading weights: 90%|█████████ | 280/311 [00:01<00:00, 250.91it/s, Materializing param=model.layers.25.mlp.gate_proj.weight] Loading weights: 91%|█████████▏| 284/311 [00:01<00:00, 247.65it/s, Materializing param=model.layers.25.self_attn.k_proj.weight] Loading weights: 59%|█████▉ | 183/311 [00:01<00:00, 172.13it/s, Materializing param=model.layers.16.post_attention_layernorm.weight] Loading weights: 95%|█████████▌| 296/311 [00:01<00:00, 261.82it/s, Materializing param=model.layers.26.self_attn.o_proj.weight] Loading weights: 90%|█████████ | 280/311 [00:01<00:00, 250.91it/s, Materializing param=model.layers.25.mlp.gate_proj.weight] Loading weights: 91%|█████████▏| 284/311 [00:01<00:00, 247.65it/s, Materializing param=model.layers.25.self_attn.k_proj.weight] Loading weights: 94%|█████████▎| 291/311 [00:01<00:00, 255.68it/s, Materializing param=model.layers.26.mlp.gate_proj.weight] Loading weights: 88%|████████▊ | 273/311 [00:01<00:00, 245.34it/s, Materializing param=model.layers.24.self_attn.k_proj.weight] Loading weights: 95%|█████████▌| 296/311 [00:01<00:00, 261.82it/s, Materializing param=model.layers.26.self_attn.o_proj.weight] Loading weights: 94%|█████████▎| 291/311 [00:01<00:00, 255.68it/s, Materializing param=model.layers.26.mlp.gate_proj.weight] Loading weights: 88%|████████▊ | 273/311 [00:01<00:00, 245.34it/s, Materializing param=model.layers.24.self_attn.k_proj.weight] Loading weights: 92%|█████████▏| 285/311 [00:01<00:00, 247.65it/s, Materializing param=model.layers.25.self_attn.o_proj.weight] Loading weights: 90%|█████████ | 281/311 [00:01<00:00, 250.91it/s, Materializing param=model.layers.25.mlp.up_proj.weight] Loading weights: 59%|█████▉ | 184/311 [00:01<00:00, 172.13it/s, Materializing param=model.layers.16.self_attn.k_norm.weight] Loading weights: 95%|█████████▌| 297/311 [00:01<00:00, 261.82it/s, Materializing param=model.layers.26.self_attn.q_norm.weight] Loading weights: 92%|█████████▏| 285/311 [00:01<00:00, 247.65it/s, Materializing param=model.layers.25.self_attn.o_proj.weight] Loading weights: 90%|█████████ | 281/311 [00:01<00:00, 250.91it/s, Materializing param=model.layers.25.mlp.up_proj.weight] Loading weights: 94%|█████████▍| 292/311 [00:01<00:00, 255.68it/s, Materializing param=model.layers.26.mlp.up_proj.weight] Loading weights: 88%|████████▊ | 274/311 [00:01<00:00, 245.34it/s, Materializing param=model.layers.24.self_attn.o_proj.weight] Loading weights: 95%|█████████▌| 297/311 [00:01<00:00, 261.82it/s, Materializing param=model.layers.26.self_attn.q_norm.weight] Loading weights: 94%|█████████▍| 292/311 [00:01<00:00, 255.68it/s, Materializing param=model.layers.26.mlp.up_proj.weight] Loading weights: 59%|█████▉ | 184/311 [00:01<00:00, 172.13it/s, Materializing param=model.layers.16.self_attn.k_norm.weight] Loading weights: 88%|████████▊ | 274/311 [00:01<00:00, 245.34it/s, Materializing param=model.layers.24.self_attn.o_proj.weight] Loading weights: 92%|█████████▏| 286/311 [00:01<00:00, 247.65it/s, Materializing param=model.layers.25.self_attn.q_norm.weight] Loading weights: 91%|█████████ | 282/311 [00:01<00:00, 250.91it/s, Materializing param=model.layers.25.post_attention_layernorm.weight] Loading weights: 96%|█████████▌| 298/311 [00:01<00:00, 261.82it/s, Materializing param=model.layers.26.self_attn.q_proj.weight] Loading weights: 92%|█████████▏| 286/311 [00:01<00:00, 247.65it/s, Materializing param=model.layers.25.self_attn.q_norm.weight] Loading weights: 91%|█████████ | 282/311 [00:01<00:00, 250.91it/s, Materializing param=model.layers.25.post_attention_layernorm.weight] Loading weights: 94%|█████████▍| 293/311 [00:01<00:00, 255.68it/s, Materializing param=model.layers.26.post_attention_layernorm.weight] Loading weights: 88%|████████▊ | 275/311 [00:01<00:00, 245.34it/s, Materializing param=model.layers.24.self_attn.q_norm.weight] Loading weights: 96%|█████████▌| 298/311 [00:01<00:00, 261.82it/s, Materializing param=model.layers.26.self_attn.q_proj.weight] Loading weights: 94%|█████████▍| 293/311 [00:01<00:00, 255.68it/s, Materializing param=model.layers.26.post_attention_layernorm.weight] Loading weights: 59%|█████▉ | 185/311 [00:01<00:00, 172.13it/s, Materializing param=model.layers.16.self_attn.k_proj.weight] Loading weights: 88%|████████▊ | 275/311 [00:01<00:00, 245.34it/s, Materializing param=model.layers.24.self_attn.q_norm.weight] Loading weights: 92%|█████████▏| 287/311 [00:01<00:00, 247.65it/s, Materializing param=model.layers.25.self_attn.q_proj.weight] Loading weights: 91%|█████████ | 283/311 [00:01<00:00, 250.91it/s, Materializing param=model.layers.25.self_attn.k_norm.weight] Loading weights: 96%|█████████▌| 299/311 [00:01<00:00, 261.82it/s, Materializing param=model.layers.26.self_attn.v_proj.weight] Loading weights: 92%|█████████▏| 287/311 [00:01<00:00, 247.65it/s, Materializing param=model.layers.25.self_attn.q_proj.weight] Loading weights: 59%|█████▉ | 185/311 [00:01<00:00, 172.13it/s, Materializing param=model.layers.16.self_attn.k_proj.weight] Loading weights: 95%|█████████▍| 294/311 [00:01<00:00, 255.68it/s, Materializing param=model.layers.26.self_attn.k_norm.weight] Loading weights: 91%|█████████ | 283/311 [00:01<00:00, 250.91it/s, Materializing param=model.layers.25.self_attn.k_norm.weight] Loading weights: 96%|█████████▌| 299/311 [00:01<00:00, 261.82it/s, Materializing param=model.layers.26.self_attn.v_proj.weight] Loading weights: 89%|████████▊ | 276/311 [00:01<00:00, 245.34it/s, Materializing param=model.layers.24.self_attn.q_proj.weight] Loading weights: 95%|█████████▍| 294/311 [00:01<00:00, 255.68it/s, Materializing param=model.layers.26.self_attn.k_norm.weight] Loading weights: 93%|█████████▎| 288/311 [00:01<00:00, 247.65it/s, Materializing param=model.layers.25.self_attn.v_proj.weight] Loading weights: 89%|████████▊ | 276/311 [00:01<00:00, 245.34it/s, Materializing param=model.layers.24.self_attn.q_proj.weight] Loading weights: 91%|█████████▏| 284/311 [00:01<00:00, 250.91it/s, Materializing param=model.layers.25.self_attn.k_proj.weight] Loading weights: 96%|█████████▋| 300/311 [00:01<00:00, 261.82it/s, Materializing param=model.layers.27.input_layernorm.weight] Loading weights: 60%|█████▉ | 186/311 [00:01<00:00, 172.13it/s, Materializing param=model.layers.16.self_attn.o_proj.weight] Loading weights: 93%|█████████▎| 288/311 [00:01<00:00, 247.65it/s, Materializing param=model.layers.25.self_attn.v_proj.weight] Loading weights: 95%|█████████▍| 295/311 [00:01<00:00, 255.68it/s, Materializing param=model.layers.26.self_attn.k_proj.weight] Loading weights: 91%|█████████▏| 284/311 [00:01<00:00, 250.91it/s, Materializing param=model.layers.25.self_attn.k_proj.weight] Loading weights: 96%|█████████▋| 300/311 [00:01<00:00, 261.82it/s, Materializing param=model.layers.27.input_layernorm.weight] Loading weights: 89%|████████▉ | 277/311 [00:01<00:00, 245.34it/s, Materializing param=model.layers.24.self_attn.v_proj.weight] Loading weights: 95%|█████████▍| 295/311 [00:01<00:00, 255.68it/s, Materializing param=model.layers.26.self_attn.k_proj.weight] Loading weights: 60%|█████▉ | 186/311 [00:01<00:00, 172.13it/s, Materializing param=model.layers.16.self_attn.o_proj.weight] Loading weights: 93%|█████████▎| 289/311 [00:01<00:00, 247.65it/s, Materializing param=model.layers.26.input_layernorm.weight] Loading weights: 89%|████████▉ | 277/311 [00:01<00:00, 245.34it/s, Materializing param=model.layers.24.self_attn.v_proj.weight] Loading weights: 92%|█████████▏| 285/311 [00:01<00:00, 250.91it/s, Materializing param=model.layers.25.self_attn.o_proj.weight] Loading weights: 93%|█████████▎| 289/311 [00:01<00:00, 247.65it/s, Materializing param=model.layers.26.input_layernorm.weight] Loading weights: 97%|█████████▋| 301/311 [00:01<00:00, 261.82it/s, Materializing param=model.layers.27.mlp.down_proj.weight] Loading weights: 95%|█████████▌| 296/311 [00:01<00:00, 255.68it/s, Materializing param=model.layers.26.self_attn.o_proj.weight] Loading weights: 92%|█████████▏| 285/311 [00:01<00:00, 250.91it/s, Materializing param=model.layers.25.self_attn.o_proj.weight] Loading weights: 97%|█████████▋| 301/311 [00:01<00:00, 261.82it/s, Materializing param=model.layers.27.mlp.down_proj.weight] Loading weights: 89%|████████▉ | 278/311 [00:01<00:00, 245.34it/s, Materializing param=model.layers.25.input_layernorm.weight] Loading weights: 95%|█████████▌| 296/311 [00:01<00:00, 255.68it/s, Materializing param=model.layers.26.self_attn.o_proj.weight] Loading weights: 93%|█████████▎| 290/311 [00:01<00:00, 247.65it/s, Materializing param=model.layers.26.mlp.down_proj.weight] Loading weights: 60%|██████ | 187/311 [00:01<00:00, 172.13it/s, Materializing param=model.layers.16.self_attn.q_norm.weight] Loading weights: 92%|█████████▏| 286/311 [00:01<00:00, 250.91it/s, Materializing param=model.layers.25.self_attn.q_norm.weight] Loading weights: 93%|█████████▎| 290/311 [00:01<00:00, 247.65it/s, Materializing param=model.layers.26.mlp.down_proj.weight] Loading weights: 97%|█████████▋| 302/311 [00:01<00:00, 261.82it/s, Materializing param=model.layers.27.mlp.gate_proj.weight] Loading weights: 89%|████████▉ | 278/311 [00:01<00:00, 245.34it/s, Materializing param=model.layers.25.input_layernorm.weight] Loading weights: 95%|█████████▌| 297/311 [00:01<00:00, 255.68it/s, Materializing param=model.layers.26.self_attn.q_norm.weight] Loading weights: 60%|██████ | 187/311 [00:01<00:00, 172.13it/s, Materializing param=model.layers.16.self_attn.q_norm.weight] Loading weights: 92%|█████████▏| 286/311 [00:01<00:00, 250.91it/s, Materializing param=model.layers.25.self_attn.q_norm.weight] Loading weights: 97%|█████████▋| 302/311 [00:01<00:00, 261.82it/s, Materializing param=model.layers.27.mlp.gate_proj.weight] Loading weights: 95%|█████████▌| 297/311 [00:01<00:00, 255.68it/s, Materializing param=model.layers.26.self_attn.q_norm.weight] Loading weights: 94%|█████████▎| 291/311 [00:01<00:00, 247.65it/s, Materializing param=model.layers.26.mlp.gate_proj.weight] Loading weights: 90%|████████▉ | 279/311 [00:01<00:00, 245.34it/s, Materializing param=model.layers.25.mlp.down_proj.weight] Loading weights: 92%|█████████▏| 287/311 [00:01<00:00, 250.91it/s, Materializing param=model.layers.25.self_attn.q_proj.weight] Loading weights: 94%|█████████▎| 291/311 [00:01<00:00, 247.65it/s, Materializing param=model.layers.26.mlp.gate_proj.weight] Loading weights: 97%|█████████▋| 303/311 [00:01<00:00, 261.82it/s, Materializing param=model.layers.27.mlp.up_proj.weight] Loading weights: 96%|█████████▌| 298/311 [00:01<00:00, 255.68it/s, Materializing param=model.layers.26.self_attn.q_proj.weight] Loading weights: 60%|██████ | 188/311 [00:01<00:00, 172.13it/s, Materializing param=model.layers.16.self_attn.q_proj.weight] Loading weights: 90%|████████▉ | 279/311 [00:01<00:00, 245.34it/s, Materializing param=model.layers.25.mlp.down_proj.weight] Loading weights: 92%|█████████▏| 287/311 [00:01<00:00, 250.91it/s, Materializing param=model.layers.25.self_attn.q_proj.weight] Loading weights: 97%|█████████▋| 303/311 [00:01<00:00, 261.82it/s, Materializing param=model.layers.27.mlp.up_proj.weight] Loading weights: 96%|█████████▌| 298/311 [00:01<00:00, 255.68it/s, Materializing param=model.layers.26.self_attn.q_proj.weight] Loading weights: 94%|█████████▍| 292/311 [00:01<00:00, 247.65it/s, Materializing param=model.layers.26.mlp.up_proj.weight] Loading weights: 60%|██████ | 188/311 [00:01<00:00, 172.13it/s, Materializing param=model.layers.16.self_attn.q_proj.weight] Loading weights: 94%|█████████▍| 292/311 [00:01<00:00, 247.65it/s, Materializing param=model.layers.26.mlp.up_proj.weight] Loading weights: 93%|█████████▎| 288/311 [00:01<00:00, 250.91it/s, Materializing param=model.layers.25.self_attn.v_proj.weight] Loading weights: 90%|█████████ | 280/311 [00:01<00:00, 245.34it/s, Materializing param=model.layers.25.mlp.gate_proj.weight] Loading weights: 98%|█████████▊| 304/311 [00:01<00:00, 261.82it/s, Materializing param=model.layers.27.post_attention_layernorm.weight] Loading weights: 96%|█████████▌| 299/311 [00:01<00:00, 255.68it/s, Materializing param=model.layers.26.self_attn.v_proj.weight] Loading weights: 93%|█████████▎| 288/311 [00:01<00:00, 250.91it/s, Materializing param=model.layers.25.self_attn.v_proj.weight] Loading weights: 90%|█████████ | 280/311 [00:01<00:00, 245.34it/s, Materializing param=model.layers.25.mlp.gate_proj.weight] Loading weights: 98%|█████████▊| 304/311 [00:01<00:00, 261.82it/s, Materializing param=model.layers.27.post_attention_layernorm.weight] Loading weights: 96%|█████████▌| 299/311 [00:01<00:00, 255.68it/s, Materializing param=model.layers.26.self_attn.v_proj.weight] Loading weights: 94%|█████████▍| 293/311 [00:01<00:00, 247.65it/s, Materializing param=model.layers.26.post_attention_layernorm.weight] Loading weights: 61%|██████ | 189/311 [00:01<00:00, 172.13it/s, Materializing param=model.layers.16.self_attn.v_proj.weight] Loading weights: 94%|█████████▍| 293/311 [00:01<00:00, 247.65it/s, Materializing param=model.layers.26.post_attention_layernorm.weight] Loading weights: 93%|█████████▎| 289/311 [00:01<00:00, 250.91it/s, Materializing param=model.layers.26.input_layernorm.weight] Loading weights: 90%|█████████ | 281/311 [00:01<00:00, 245.34it/s, Materializing param=model.layers.25.mlp.up_proj.weight] Loading weights: 98%|█████████▊| 305/311 [00:01<00:00, 261.82it/s, Materializing param=model.layers.27.self_attn.k_norm.weight] Loading weights: 96%|█████████▋| 300/311 [00:01<00:00, 255.68it/s, Materializing param=model.layers.27.input_layernorm.weight] Loading weights: 90%|█████████ | 281/311 [00:01<00:00, 245.34it/s, Materializing param=model.layers.25.mlp.up_proj.weight] Loading weights: 61%|██████ | 189/311 [00:01<00:00, 172.13it/s, Materializing param=model.layers.16.self_attn.v_proj.weight] Loading weights: 93%|█████████▎| 289/311 [00:01<00:00, 250.91it/s, Materializing param=model.layers.26.input_layernorm.weight] Loading weights: 98%|█████████▊| 305/311 [00:01<00:00, 261.82it/s, Materializing param=model.layers.27.self_attn.k_norm.weight] Loading weights: 96%|█████████▋| 300/311 [00:01<00:00, 255.68it/s, Materializing param=model.layers.27.input_layernorm.weight] Loading weights: 95%|█████████▍| 294/311 [00:01<00:00, 247.65it/s, Materializing param=model.layers.26.self_attn.k_norm.weight] Loading weights: 95%|█████████▍| 294/311 [00:01<00:00, 247.65it/s, Materializing param=model.layers.26.self_attn.k_norm.weight] Loading weights: 91%|█████████ | 282/311 [00:01<00:00, 245.34it/s, Materializing param=model.layers.25.post_attention_layernorm.weight] Loading weights: 98%|█████████▊| 306/311 [00:01<00:00, 261.82it/s, Materializing param=model.layers.27.self_attn.k_proj.weight] Loading weights: 93%|█████████▎| 290/311 [00:01<00:00, 250.91it/s, Materializing param=model.layers.26.mlp.down_proj.weight] Loading weights: 97%|█████████▋| 301/311 [00:01<00:00, 255.68it/s, Materializing param=model.layers.27.mlp.down_proj.weight] Loading weights: 61%|██████ | 190/311 [00:01<00:00, 172.13it/s, Materializing param=model.layers.17.input_layernorm.weight] Loading weights: 91%|█████████ | 282/311 [00:01<00:00, 245.34it/s, Materializing param=model.layers.25.post_attention_layernorm.weight] Loading weights: 98%|█████████▊| 306/311 [00:01<00:00, 261.82it/s, Materializing param=model.layers.27.self_attn.k_proj.weight] Loading weights: 93%|█████████▎| 290/311 [00:01<00:00, 250.91it/s, Materializing param=model.layers.26.mlp.down_proj.weight] Loading weights: 97%|█████████▋| 301/311 [00:01<00:00, 255.68it/s, Materializing param=model.layers.27.mlp.down_proj.weight] Loading weights: 95%|█████████▍| 295/311 [00:01<00:00, 247.65it/s, Materializing param=model.layers.26.self_attn.k_proj.weight] Loading weights: 61%|██████ | 190/311 [00:01<00:00, 172.13it/s, Materializing param=model.layers.17.input_layernorm.weight] Loading weights: 95%|█████████▍| 295/311 [00:01<00:00, 247.65it/s, Materializing param=model.layers.26.self_attn.k_proj.weight] Loading weights: 91%|█████████ | 283/311 [00:01<00:00, 245.34it/s, Materializing param=model.layers.25.self_attn.k_norm.weight] Loading weights: 97%|█████████▋| 302/311 [00:01<00:00, 255.68it/s, Materializing param=model.layers.27.mlp.gate_proj.weight] Loading weights: 99%|█████████▊| 307/311 [00:01<00:00, 261.82it/s, Materializing param=model.layers.27.self_attn.o_proj.weight] Loading weights: 94%|█████████▎| 291/311 [00:01<00:00, 250.91it/s, Materializing param=model.layers.26.mlp.gate_proj.weight] Loading weights: 97%|█████████▋| 302/311 [00:01<00:00, 255.68it/s, Materializing param=model.layers.27.mlp.gate_proj.weight] Loading weights: 91%|█████████ | 283/311 [00:01<00:00, 245.34it/s, Materializing param=model.layers.25.self_attn.k_norm.weight] Loading weights: 99%|█████████▊| 307/311 [00:01<00:00, 261.82it/s, Materializing param=model.layers.27.self_attn.o_proj.weight] Loading weights: 94%|█████████▎| 291/311 [00:01<00:00, 250.91it/s, Materializing param=model.layers.26.mlp.gate_proj.weight] Loading weights: 95%|█████████▌| 296/311 [00:01<00:00, 247.65it/s, Materializing param=model.layers.26.self_attn.o_proj.weight] Loading weights: 61%|██████▏ | 191/311 [00:01<00:00, 172.13it/s, Materializing param=model.layers.17.mlp.down_proj.weight] Loading weights: 95%|█████████▌| 296/311 [00:01<00:00, 247.65it/s, Materializing param=model.layers.26.self_attn.o_proj.weight] Loading weights: 97%|█████████▋| 303/311 [00:01<00:00, 255.68it/s, Materializing param=model.layers.27.mlp.up_proj.weight] Loading weights: 99%|█████████▉| 308/311 [00:01<00:00, 261.82it/s, Materializing param=model.layers.27.self_attn.q_norm.weight] Loading weights: 91%|█████████▏| 284/311 [00:01<00:00, 245.34it/s, Materializing param=model.layers.25.self_attn.k_proj.weight] Loading weights: 94%|█████████▍| 292/311 [00:01<00:00, 250.91it/s, Materializing param=model.layers.26.mlp.up_proj.weight] Loading weights: 61%|██████▏ | 191/311 [00:01<00:00, 172.13it/s, Materializing param=model.layers.17.mlp.down_proj.weight] Loading weights: 97%|█████████▋| 303/311 [00:01<00:00, 255.68it/s, Materializing param=model.layers.27.mlp.up_proj.weight] Loading weights: 99%|█████████▉| 308/311 [00:01<00:00, 261.82it/s, Materializing param=model.layers.27.self_attn.q_norm.weight] Loading weights: 91%|█████████▏| 284/311 [00:01<00:00, 245.34it/s, Materializing param=model.layers.25.self_attn.k_proj.weight] Loading weights: 94%|█████████▍| 292/311 [00:01<00:00, 250.91it/s, Materializing param=model.layers.26.mlp.up_proj.weight] Loading weights: 95%|█████████▌| 297/311 [00:01<00:00, 247.65it/s, Materializing param=model.layers.26.self_attn.q_norm.weight] Loading weights: 95%|█████████▌| 297/311 [00:01<00:00, 247.65it/s, Materializing param=model.layers.26.self_attn.q_norm.weight] Loading weights: 98%|█████████▊| 304/311 [00:01<00:00, 255.68it/s, Materializing param=model.layers.27.post_attention_layernorm.weight] Loading weights: 99%|█████████▉| 309/311 [00:01<00:00, 261.82it/s, Materializing param=model.layers.27.self_attn.q_proj.weight] Loading weights: 92%|█████████▏| 285/311 [00:01<00:00, 245.34it/s, Materializing param=model.layers.25.self_attn.o_proj.weight] Loading weights: 94%|█████████▍| 293/311 [00:01<00:00, 250.91it/s, Materializing param=model.layers.26.post_attention_layernorm.weight] Loading weights: 62%|██████▏ | 192/311 [00:01<00:00, 172.13it/s, Materializing param=model.layers.17.mlp.gate_proj.weight] Loading weights: 98%|█████████▊| 304/311 [00:01<00:00, 255.68it/s, Materializing param=model.layers.27.post_attention_layernorm.weight] Loading weights: 99%|█████████▉| 309/311 [00:01<00:00, 261.82it/s, Materializing param=model.layers.27.self_attn.q_proj.weight] Loading weights: 92%|█████████▏| 285/311 [00:01<00:00, 245.34it/s, Materializing param=model.layers.25.self_attn.o_proj.weight] Loading weights: 96%|█████████▌| 298/311 [00:01<00:00, 247.65it/s, Materializing param=model.layers.26.self_attn.q_proj.weight] Loading weights: 94%|█████████▍| 293/311 [00:01<00:00, 250.91it/s, Materializing param=model.layers.26.post_attention_layernorm.weight] Loading weights: 62%|██████▏ | 192/311 [00:01<00:00, 172.13it/s, Materializing param=model.layers.17.mlp.gate_proj.weight] Loading weights: 96%|█████████▌| 298/311 [00:01<00:00, 247.65it/s, Materializing param=model.layers.26.self_attn.q_proj.weight] Loading weights: 98%|█████████▊| 305/311 [00:01<00:00, 255.68it/s, Materializing param=model.layers.27.self_attn.k_norm.weight] Loading weights: 100%|█████████▉| 310/311 [00:01<00:00, 261.82it/s, Materializing param=model.layers.27.self_attn.v_proj.weight] Loading weights: 92%|█████████▏| 286/311 [00:01<00:00, 245.34it/s, Materializing param=model.layers.25.self_attn.q_norm.weight] Loading weights: 95%|█████████▍| 294/311 [00:01<00:00, 250.91it/s, Materializing param=model.layers.26.self_attn.k_norm.weight] Loading weights: 98%|█████████▊| 305/311 [00:01<00:00, 255.68it/s, Materializing param=model.layers.27.self_attn.k_norm.weight] Loading weights: 100%|█████████▉| 310/311 [00:01<00:00, 261.82it/s, Materializing param=model.layers.27.self_attn.v_proj.weight] Loading weights: 92%|█████████▏| 286/311 [00:01<00:00, 245.34it/s, Materializing param=model.layers.25.self_attn.q_norm.weight] Loading weights: 96%|█████████▌| 299/311 [00:01<00:00, 247.65it/s, Materializing param=model.layers.26.self_attn.v_proj.weight] Loading weights: 95%|█████████▍| 294/311 [00:01<00:00, 250.91it/s, Materializing param=model.layers.26.self_attn.k_norm.weight] Loading weights: 96%|█████████▌| 299/311 [00:01<00:00, 247.65it/s, Materializing param=model.layers.26.self_attn.v_proj.weight] Loading weights: 62%|██████▏ | 193/311 [00:01<00:00, 172.13it/s, Materializing param=model.layers.17.mlp.up_proj.weight] Loading weights: 98%|█████████▊| 306/311 [00:01<00:00, 255.68it/s, Materializing param=model.layers.27.self_attn.k_proj.weight] Loading weights: 100%|██████████| 311/311 [00:01<00:00, 261.82it/s, Materializing param=model.norm.weight] Loading weights: 92%|█████████▏| 287/311 [00:01<00:00, 245.34it/s, Materializing param=model.layers.25.self_attn.q_proj.weight] Loading weights: 95%|█████████▍| 295/311 [00:01<00:00, 250.91it/s, Materializing param=model.layers.26.self_attn.k_proj.weight] Loading weights: 98%|█████████▊| 306/311 [00:01<00:00, 255.68it/s, Materializing param=model.layers.27.self_attn.k_proj.weight] Loading weights: 100%|██████████| 311/311 [00:01<00:00, 261.82it/s, Materializing param=model.norm.weight] Loading weights: 62%|██████▏ | 193/311 [00:01<00:00, 172.13it/s, Materializing param=model.layers.17.mlp.up_proj.weight] Loading weights: 96%|█████████▋| 300/311 [00:01<00:00, 247.65it/s, Materializing param=model.layers.27.input_layernorm.weight] Loading weights: 92%|█████████▏| 287/311 [00:01<00:00, 245.34it/s, Materializing param=model.layers.25.self_attn.q_proj.weight] Loading weights: 95%|█████████▍| 295/311 [00:01<00:00, 250.91it/s, Materializing param=model.layers.26.self_attn.k_proj.weight] Loading weights: 96%|█████████▋| 300/311 [00:01<00:00, 247.65it/s, Materializing param=model.layers.27.input_layernorm.weight] Loading weights: 99%|█████████▊| 307/311 [00:01<00:00, 255.68it/s, Materializing param=model.layers.27.self_attn.o_proj.weight] Loading weights: 93%|█████████▎| 288/311 [00:01<00:00, 245.34it/s, Materializing param=model.layers.25.self_attn.v_proj.weight] Loading weights: 100%|██████████| 311/311 [00:01<00:00, 175.99it/s, Materializing param=model.norm.weight] + Loading weights: 95%|█████████▌| 296/311 [00:01<00:00, 250.91it/s, Materializing param=model.layers.26.self_attn.o_proj.weight] Loading weights: 99%|█████████▊| 307/311 [00:01<00:00, 255.68it/s, Materializing param=model.layers.27.self_attn.o_proj.weight] Loading weights: 62%|██████▏ | 194/311 [00:01<00:00, 172.13it/s, Materializing param=model.layers.17.post_attention_layernorm.weight] Loading weights: 97%|█████████▋| 301/311 [00:01<00:00, 247.65it/s, Materializing param=model.layers.27.mlp.down_proj.weight] Loading weights: 93%|█████████▎| 288/311 [00:01<00:00, 245.34it/s, Materializing param=model.layers.25.self_attn.v_proj.weight] Loading weights: 95%|█████████▌| 296/311 [00:01<00:00, 250.91it/s, Materializing param=model.layers.26.self_attn.o_proj.weight] Loading weights: 97%|█████████▋| 301/311 [00:01<00:00, 247.65it/s, Materializing param=model.layers.27.mlp.down_proj.weight] Loading weights: 99%|█████████▉| 308/311 [00:01<00:00, 255.68it/s, Materializing param=model.layers.27.self_attn.q_norm.weight] Loading weights: 62%|██████▏ | 194/311 [00:01<00:00, 172.13it/s, Materializing param=model.layers.17.post_attention_layernorm.weight] Loading weights: 93%|█████████▎| 289/311 [00:01<00:00, 245.34it/s, Materializing param=model.layers.26.input_layernorm.weight] Loading weights: 99%|█████████▉| 308/311 [00:01<00:00, 255.68it/s, Materializing param=model.layers.27.self_attn.q_norm.weight] Loading weights: 95%|█████████▌| 297/311 [00:01<00:00, 250.91it/s, Materializing param=model.layers.26.self_attn.q_norm.weight] Loading weights: 97%|█████████▋| 302/311 [00:01<00:00, 247.65it/s, Materializing param=model.layers.27.mlp.gate_proj.weight] Loading weights: 93%|█████████▎| 289/311 [00:01<00:00, 245.34it/s, Materializing param=model.layers.26.input_layernorm.weight] Loading weights: 95%|█████████▌| 297/311 [00:01<00:00, 250.91it/s, Materializing param=model.layers.26.self_attn.q_norm.weight] Loading weights: 97%|█████████▋| 302/311 [00:01<00:00, 247.65it/s, Materializing param=model.layers.27.mlp.gate_proj.weight] Loading weights: 63%|██████▎ | 195/311 [00:01<00:00, 172.13it/s, Materializing param=model.layers.17.self_attn.k_norm.weight] Loading weights: 99%|█████████▉| 309/311 [00:01<00:00, 255.68it/s, Materializing param=model.layers.27.self_attn.q_proj.weight] Loading weights: 99%|█████████▉| 309/311 [00:01<00:00, 255.68it/s, Materializing param=model.layers.27.self_attn.q_proj.weight] Loading weights: 96%|█████████▌| 298/311 [00:01<00:00, 250.91it/s, Materializing param=model.layers.26.self_attn.q_proj.weight] Loading weights: 97%|█████████▋| 303/311 [00:01<00:00, 247.65it/s, Materializing param=model.layers.27.mlp.up_proj.weight] Loading weights: 63%|██████▎ | 195/311 [00:01<00:00, 172.13it/s, Materializing param=model.layers.17.self_attn.k_norm.weight] Loading weights: 93%|█████████▎| 290/311 [00:01<00:00, 245.34it/s, Materializing param=model.layers.26.mlp.down_proj.weight] Loading weights: 96%|█████████▌| 298/311 [00:01<00:00, 250.91it/s, Materializing param=model.layers.26.self_attn.q_proj.weight] Loading weights: 97%|█████████▋| 303/311 [00:01<00:00, 247.65it/s, Materializing param=model.layers.27.mlp.up_proj.weight] Loading weights: 100%|█████████▉| 310/311 [00:01<00:00, 255.68it/s, Materializing param=model.layers.27.self_attn.v_proj.weight] Loading weights: 93%|█████████▎| 290/311 [00:01<00:00, 245.34it/s, Materializing param=model.layers.26.mlp.down_proj.weight] Loading weights: 100%|█████████▉| 310/311 [00:01<00:00, 255.68it/s, Materializing param=model.layers.27.self_attn.v_proj.weight] Loading weights: 63%|██████▎ | 196/311 [00:01<00:00, 172.13it/s, Materializing param=model.layers.17.self_attn.k_proj.weight] Loading weights: 96%|█████████▌| 299/311 [00:01<00:00, 250.91it/s, Materializing param=model.layers.26.self_attn.v_proj.weight] Loading weights: 98%|█████████▊| 304/311 [00:01<00:00, 247.65it/s, Materializing param=model.layers.27.post_attention_layernorm.weight] Loading weights: 96%|█████████▌| 299/311 [00:01<00:00, 250.91it/s, Materializing param=model.layers.26.self_attn.v_proj.weight] Loading weights: 98%|█████████▊| 304/311 [00:01<00:00, 247.65it/s, Materializing param=model.layers.27.post_attention_layernorm.weight] Loading weights: 94%|█████████▎| 291/311 [00:01<00:00, 245.34it/s, Materializing param=model.layers.26.mlp.gate_proj.weight] Loading weights: 63%|██████▎ | 196/311 [00:01<00:00, 172.13it/s, Materializing param=model.layers.17.self_attn.k_proj.weight] Loading weights: 100%|██████████| 311/311 [00:01<00:00, 255.68it/s, Materializing param=model.norm.weight] Loading weights: 94%|█████████▎| 291/311 [00:01<00:00, 245.34it/s, Materializing param=model.layers.26.mlp.gate_proj.weight] Loading weights: 100%|██████████| 311/311 [00:01<00:00, 255.68it/s, Materializing param=model.norm.weight] Loading weights: 96%|█████████▋| 300/311 [00:01<00:00, 250.91it/s, Materializing param=model.layers.27.input_layernorm.weight] Loading weights: 98%|█████████▊| 305/311 [00:01<00:00, 247.65it/s, Materializing param=model.layers.27.self_attn.k_norm.weight] Loading weights: 98%|█████████▊| 305/311 [00:01<00:00, 247.65it/s, Materializing param=model.layers.27.self_attn.k_norm.weight] Loading weights: 96%|█████████▋| 300/311 [00:01<00:00, 250.91it/s, Materializing param=model.layers.27.input_layernorm.weight] Loading weights: 94%|█████████▍| 292/311 [00:01<00:00, 245.34it/s, Materializing param=model.layers.26.mlp.up_proj.weight] Loading weights: 63%|██████▎ | 197/311 [00:01<00:00, 172.13it/s, Materializing param=model.layers.17.self_attn.o_proj.weight] Loading weights: 100%|██████████| 311/311 [00:01<00:00, 170.80it/s, Materializing param=model.norm.weight] Loading weights: 94%|█████████▍| 292/311 [00:01<00:00, 245.34it/s, Materializing param=model.layers.26.mlp.up_proj.weight] + Loading weights: 63%|██████▎ | 197/311 [00:01<00:00, 172.13it/s, Materializing param=model.layers.17.self_attn.o_proj.weight] Loading weights: 98%|█████████▊| 306/311 [00:01<00:00, 247.65it/s, Materializing param=model.layers.27.self_attn.k_proj.weight] Loading weights: 97%|█████████▋| 301/311 [00:01<00:00, 250.91it/s, Materializing param=model.layers.27.mlp.down_proj.weight] Loading weights: 98%|█████████▊| 306/311 [00:01<00:00, 247.65it/s, Materializing param=model.layers.27.self_attn.k_proj.weight] Loading weights: 97%|█████████▋| 301/311 [00:01<00:00, 250.91it/s, Materializing param=model.layers.27.mlp.down_proj.weight] Loading weights: 94%|█████████▍| 293/311 [00:01<00:00, 245.34it/s, Materializing param=model.layers.26.post_attention_layernorm.weight] Loading weights: 94%|█████████▍| 293/311 [00:01<00:00, 245.34it/s, Materializing param=model.layers.26.post_attention_layernorm.weight] Loading weights: 64%|██████▎ | 198/311 [00:01<00:00, 172.13it/s, Materializing param=model.layers.17.self_attn.q_norm.weight] Loading weights: 99%|█████████▊| 307/311 [00:01<00:00, 247.65it/s, Materializing param=model.layers.27.self_attn.o_proj.weight] Loading weights: 97%|█████████▋| 302/311 [00:01<00:00, 250.91it/s, Materializing param=model.layers.27.mlp.gate_proj.weight] Loading weights: 64%|██████▎ | 198/311 [00:01<00:00, 172.13it/s, Materializing param=model.layers.17.self_attn.q_norm.weight] Loading weights: 99%|█████████▊| 307/311 [00:01<00:00, 247.65it/s, Materializing param=model.layers.27.self_attn.o_proj.weight] Loading weights: 97%|█████████▋| 302/311 [00:01<00:00, 250.91it/s, Materializing param=model.layers.27.mlp.gate_proj.weight] Loading weights: 95%|█████████▍| 294/311 [00:01<00:00, 245.34it/s, Materializing param=model.layers.26.self_attn.k_norm.weight] Loading weights: 95%|█████████▍| 294/311 [00:01<00:00, 245.34it/s, Materializing param=model.layers.26.self_attn.k_norm.weight] Loading weights: 99%|█████████▉| 308/311 [00:01<00:00, 247.65it/s, Materializing param=model.layers.27.self_attn.q_norm.weight] Loading weights: 97%|█████████▋| 303/311 [00:01<00:00, 250.91it/s, Materializing param=model.layers.27.mlp.up_proj.weight] Loading weights: 64%|██████▍ | 199/311 [00:01<00:00, 172.13it/s, Materializing param=model.layers.17.self_attn.q_proj.weight] Loading weights: 99%|█████████▉| 308/311 [00:01<00:00, 247.65it/s, Materializing param=model.layers.27.self_attn.q_norm.weight] Loading weights: 97%|█████████▋| 303/311 [00:01<00:00, 250.91it/s, Materializing param=model.layers.27.mlp.up_proj.weight] Loading weights: 95%|█████████▍| 295/311 [00:01<00:00, 245.34it/s, Materializing param=model.layers.26.self_attn.k_proj.weight] Loading weights: 64%|██████▍ | 199/311 [00:01<00:00, 172.13it/s, Materializing param=model.layers.17.self_attn.q_proj.weight] Loading weights: 95%|█████████▍| 295/311 [00:01<00:00, 245.34it/s, Materializing param=model.layers.26.self_attn.k_proj.weight] Loading weights: 99%|█████████▉| 309/311 [00:01<00:00, 247.65it/s, Materializing param=model.layers.27.self_attn.q_proj.weight] Loading weights: 98%|█████████▊| 304/311 [00:01<00:00, 250.91it/s, Materializing param=model.layers.27.post_attention_layernorm.weight] Loading weights: 99%|█████████▉| 309/311 [00:01<00:00, 247.65it/s, Materializing param=model.layers.27.self_attn.q_proj.weight] Loading weights: 98%|█████████▊| 304/311 [00:01<00:00, 250.91it/s, Materializing param=model.layers.27.post_attention_layernorm.weight] Loading weights: 95%|█████████▌| 296/311 [00:01<00:00, 245.34it/s, Materializing param=model.layers.26.self_attn.o_proj.weight] Loading weights: 64%|██████▍ | 200/311 [00:01<00:00, 172.13it/s, Materializing param=model.layers.17.self_attn.v_proj.weight] Loading weights: 95%|█████████▌| 296/311 [00:01<00:00, 245.34it/s, Materializing param=model.layers.26.self_attn.o_proj.weight] Loading weights: 100%|█████████▉| 310/311 [00:01<00:00, 247.65it/s, Materializing param=model.layers.27.self_attn.v_proj.weight] Loading weights: 98%|█████████▊| 305/311 [00:01<00:00, 250.91it/s, Materializing param=model.layers.27.self_attn.k_norm.weight] Loading weights: 64%|██████▍ | 200/311 [00:01<00:00, 172.13it/s, Materializing param=model.layers.17.self_attn.v_proj.weight] Loading weights: 100%|█████████▉| 310/311 [00:01<00:00, 247.65it/s, Materializing param=model.layers.27.self_attn.v_proj.weight] Loading weights: 98%|█████████▊| 305/311 [00:01<00:00, 250.91it/s, Materializing param=model.layers.27.self_attn.k_norm.weight] Loading weights: 95%|█████████▌| 297/311 [00:01<00:00, 245.34it/s, Materializing param=model.layers.26.self_attn.q_norm.weight] Loading weights: 95%|█████████▌| 297/311 [00:01<00:00, 245.34it/s, Materializing param=model.layers.26.self_attn.q_norm.weight] Loading weights: 100%|██████████| 311/311 [00:01<00:00, 247.65it/s, Materializing param=model.norm.weight] Loading weights: 65%|██████▍ | 201/311 [00:01<00:00, 172.13it/s, Materializing param=model.layers.18.input_layernorm.weight] Loading weights: 98%|█████████▊| 306/311 [00:01<00:00, 250.91it/s, Materializing param=model.layers.27.self_attn.k_proj.weight] Loading weights: 100%|██████████| 311/311 [00:01<00:00, 247.65it/s, Materializing param=model.norm.weight] Loading weights: 98%|█████████▊| 306/311 [00:01<00:00, 250.91it/s, Materializing param=model.layers.27.self_attn.k_proj.weight] Loading weights: 65%|██████▍ | 201/311 [00:01<00:00, 172.13it/s, Materializing param=model.layers.18.input_layernorm.weight] Loading weights: 96%|█████████▌| 298/311 [00:01<00:00, 245.34it/s, Materializing param=model.layers.26.self_attn.q_proj.weight] Loading weights: 99%|█████████▊| 307/311 [00:01<00:00, 250.91it/s, Materializing param=model.layers.27.self_attn.o_proj.weight] Loading weights: 100%|██████████| 311/311 [00:01<00:00, 175.84it/s, Materializing param=model.norm.weight] + Loading weights: 96%|█████████▌| 298/311 [00:01<00:00, 245.34it/s, Materializing param=model.layers.26.self_attn.q_proj.weight] Loading weights: 99%|█████████▊| 307/311 [00:01<00:00, 250.91it/s, Materializing param=model.layers.27.self_attn.o_proj.weight] Loading weights: 65%|██████▍ | 202/311 [00:01<00:00, 172.13it/s, Materializing param=model.layers.18.mlp.down_proj.weight] Loading weights: 65%|██████▍ | 202/311 [00:01<00:00, 172.13it/s, Materializing param=model.layers.18.mlp.down_proj.weight] Loading weights: 99%|█████████▉| 308/311 [00:01<00:00, 250.91it/s, Materializing param=model.layers.27.self_attn.q_norm.weight] Loading weights: 96%|█████████▌| 299/311 [00:01<00:00, 245.34it/s, Materializing param=model.layers.26.self_attn.v_proj.weight] Loading weights: 99%|█████████▉| 308/311 [00:01<00:00, 250.91it/s, Materializing param=model.layers.27.self_attn.q_norm.weight] Loading weights: 96%|█████████▌| 299/311 [00:01<00:00, 245.34it/s, Materializing param=model.layers.26.self_attn.v_proj.weight] Loading weights: 65%|██████▌ | 203/311 [00:01<00:00, 172.13it/s, Materializing param=model.layers.18.mlp.gate_proj.weight] Loading weights: 99%|█████████▉| 309/311 [00:01<00:00, 250.91it/s, Materializing param=model.layers.27.self_attn.q_proj.weight] Loading weights: 96%|█████████▋| 300/311 [00:01<00:00, 245.34it/s, Materializing param=model.layers.27.input_layernorm.weight] Loading weights: 65%|██████▌ | 203/311 [00:01<00:00, 172.13it/s, Materializing param=model.layers.18.mlp.gate_proj.weight] Loading weights: 99%|█████████▉| 309/311 [00:01<00:00, 250.91it/s, Materializing param=model.layers.27.self_attn.q_proj.weight] Loading weights: 96%|█████████▋| 300/311 [00:01<00:00, 245.34it/s, Materializing param=model.layers.27.input_layernorm.weight] Loading weights: 66%|██████▌ | 204/311 [00:01<00:00, 172.13it/s, Materializing param=model.layers.18.mlp.up_proj.weight] Loading weights: 100%|█████████▉| 310/311 [00:01<00:00, 250.91it/s, Materializing param=model.layers.27.self_attn.v_proj.weight] Loading weights: 97%|█████████▋| 301/311 [00:01<00:00, 245.34it/s, Materializing param=model.layers.27.mlp.down_proj.weight] Loading weights: 100%|█████████▉| 310/311 [00:01<00:00, 250.91it/s, Materializing param=model.layers.27.self_attn.v_proj.weight] Loading weights: 66%|██████▌ | 204/311 [00:01<00:00, 172.13it/s, Materializing param=model.layers.18.mlp.up_proj.weight] Loading weights: 97%|█████████▋| 301/311 [00:01<00:00, 245.34it/s, Materializing param=model.layers.27.mlp.down_proj.weight] Loading weights: 100%|██████████| 311/311 [00:01<00:00, 250.91it/s, Materializing param=model.norm.weight] Loading weights: 97%|█████████▋| 302/311 [00:01<00:00, 245.34it/s, Materializing param=model.layers.27.mlp.gate_proj.weight] Loading weights: 66%|██████▌ | 205/311 [00:01<00:00, 172.13it/s, Materializing param=model.layers.18.post_attention_layernorm.weight] Loading weights: 100%|██████████| 311/311 [00:01<00:00, 250.91it/s, Materializing param=model.norm.weight] Loading weights: 97%|█████████▋| 302/311 [00:01<00:00, 245.34it/s, Materializing param=model.layers.27.mlp.gate_proj.weight] Loading weights: 66%|██████▌ | 205/311 [00:01<00:00, 172.13it/s, Materializing param=model.layers.18.post_attention_layernorm.weight] Loading weights: 97%|█████████▋| 303/311 [00:01<00:00, 245.34it/s, Materializing param=model.layers.27.mlp.up_proj.weight] Loading weights: 100%|██████████| 311/311 [00:01<00:00, 175.36it/s, Materializing param=model.norm.weight] + Loading weights: 97%|█████████▋| 303/311 [00:01<00:00, 245.34it/s, Materializing param=model.layers.27.mlp.up_proj.weight] Loading weights: 66%|██████▌ | 206/311 [00:01<00:00, 172.13it/s, Materializing param=model.layers.18.self_attn.k_norm.weight] Loading weights: 66%|██████▌ | 206/311 [00:01<00:00, 172.13it/s, Materializing param=model.layers.18.self_attn.k_norm.weight] Loading weights: 98%|█████████▊| 304/311 [00:01<00:00, 245.34it/s, Materializing param=model.layers.27.post_attention_layernorm.weight] Loading weights: 98%|█████████▊| 304/311 [00:01<00:00, 245.34it/s, Materializing param=model.layers.27.post_attention_layernorm.weight] Loading weights: 67%|██████▋ | 207/311 [00:01<00:00, 172.13it/s, Materializing param=model.layers.18.self_attn.k_proj.weight] Loading weights: 98%|█████████▊| 305/311 [00:01<00:00, 245.34it/s, Materializing param=model.layers.27.self_attn.k_norm.weight] Loading weights: 67%|██████▋ | 207/311 [00:01<00:00, 172.13it/s, Materializing param=model.layers.18.self_attn.k_proj.weight] Loading weights: 98%|█████████▊| 305/311 [00:01<00:00, 245.34it/s, Materializing param=model.layers.27.self_attn.k_norm.weight] Loading weights: 67%|██████▋ | 208/311 [00:01<00:00, 172.13it/s, Materializing param=model.layers.18.self_attn.o_proj.weight] Loading weights: 98%|█████████▊| 306/311 [00:01<00:00, 245.34it/s, Materializing param=model.layers.27.self_attn.k_proj.weight] Loading weights: 98%|█████████▊| 306/311 [00:01<00:00, 245.34it/s, Materializing param=model.layers.27.self_attn.k_proj.weight] Loading weights: 67%|██████▋ | 208/311 [00:01<00:00, 172.13it/s, Materializing param=model.layers.18.self_attn.o_proj.weight] Loading weights: 99%|█████████▊| 307/311 [00:01<00:00, 245.34it/s, Materializing param=model.layers.27.self_attn.o_proj.weight] Loading weights: 67%|██████▋ | 209/311 [00:01<00:00, 172.13it/s, Materializing param=model.layers.18.self_attn.q_norm.weight] Loading weights: 99%|█████████▊| 307/311 [00:01<00:00, 245.34it/s, Materializing param=model.layers.27.self_attn.o_proj.weight] Loading weights: 67%|██████▋ | 209/311 [00:01<00:00, 172.13it/s, Materializing param=model.layers.18.self_attn.q_norm.weight] Loading weights: 99%|█████████▉| 308/311 [00:01<00:00, 245.34it/s, Materializing param=model.layers.27.self_attn.q_norm.weight] Loading weights: 99%|█████████▉| 308/311 [00:01<00:00, 245.34it/s, Materializing param=model.layers.27.self_attn.q_norm.weight] Loading weights: 68%|██████▊ | 210/311 [00:01<00:00, 172.13it/s, Materializing param=model.layers.18.self_attn.q_proj.weight] Loading weights: 68%|██████▊ | 210/311 [00:01<00:00, 172.13it/s, Materializing param=model.layers.18.self_attn.q_proj.weight] Loading weights: 99%|█████████▉| 309/311 [00:01<00:00, 245.34it/s, Materializing param=model.layers.27.self_attn.q_proj.weight] Loading weights: 99%|█████████▉| 309/311 [00:01<00:00, 245.34it/s, Materializing param=model.layers.27.self_attn.q_proj.weight] Loading weights: 68%|██████▊ | 211/311 [00:01<00:00, 172.13it/s, Materializing param=model.layers.18.self_attn.v_proj.weight] Loading weights: 100%|█████████▉| 310/311 [00:01<00:00, 245.34it/s, Materializing param=model.layers.27.self_attn.v_proj.weight] Loading weights: 68%|██████▊ | 211/311 [00:01<00:00, 172.13it/s, Materializing param=model.layers.18.self_attn.v_proj.weight] Loading weights: 100%|█████████▉| 310/311 [00:01<00:00, 245.34it/s, Materializing param=model.layers.27.self_attn.v_proj.weight] Loading weights: 100%|██████████| 311/311 [00:01<00:00, 245.34it/s, Materializing param=model.norm.weight] Loading weights: 68%|██████▊ | 212/311 [00:01<00:00, 172.13it/s, Materializing param=model.layers.19.input_layernorm.weight] Loading weights: 100%|██████████| 311/311 [00:01<00:00, 245.34it/s, Materializing param=model.norm.weight] Loading weights: 68%|██████▊ | 212/311 [00:01<00:00, 172.13it/s, Materializing param=model.layers.19.input_layernorm.weight] Loading weights: 100%|██████████| 311/311 [00:01<00:00, 173.28it/s, Materializing param=model.norm.weight] + Loading weights: 68%|██████▊ | 213/311 [00:01<00:00, 172.13it/s, Materializing param=model.layers.19.mlp.down_proj.weight] Loading weights: 68%|██████▊ | 213/311 [00:01<00:00, 172.13it/s, Materializing param=model.layers.19.mlp.down_proj.weight] Loading weights: 69%|██████▉ | 214/311 [00:01<00:00, 172.13it/s, Materializing param=model.layers.19.mlp.gate_proj.weight] Loading weights: 69%|██████▉ | 214/311 [00:01<00:00, 172.13it/s, Materializing param=model.layers.19.mlp.gate_proj.weight] Loading weights: 69%|██████▉ | 215/311 [00:01<00:00, 172.13it/s, Materializing param=model.layers.19.mlp.up_proj.weight] Loading weights: 69%|██████▉ | 215/311 [00:01<00:00, 172.13it/s, Materializing param=model.layers.19.mlp.up_proj.weight] Loading weights: 69%|██████▉ | 216/311 [00:01<00:00, 172.13it/s, Materializing param=model.layers.19.post_attention_layernorm.weight] Loading weights: 69%|██████▉ | 216/311 [00:01<00:00, 172.13it/s, Materializing param=model.layers.19.post_attention_layernorm.weight] Loading weights: 70%|██████▉ | 217/311 [00:01<00:00, 172.13it/s, Materializing param=model.layers.19.self_attn.k_norm.weight] Loading weights: 70%|██████▉ | 217/311 [00:01<00:00, 172.13it/s, Materializing param=model.layers.19.self_attn.k_norm.weight] Loading weights: 70%|███████ | 218/311 [00:01<00:00, 172.13it/s, Materializing param=model.layers.19.self_attn.k_proj.weight] Loading weights: 70%|███████ | 218/311 [00:01<00:00, 172.13it/s, Materializing param=model.layers.19.self_attn.k_proj.weight] Loading weights: 70%|███████ | 219/311 [00:01<00:00, 172.13it/s, Materializing param=model.layers.19.self_attn.o_proj.weight] Loading weights: 70%|███████ | 219/311 [00:01<00:00, 172.13it/s, Materializing param=model.layers.19.self_attn.o_proj.weight] Loading weights: 71%|███████ | 220/311 [00:01<00:00, 172.13it/s, Materializing param=model.layers.19.self_attn.q_norm.weight] Loading weights: 71%|███████ | 220/311 [00:01<00:00, 172.13it/s, Materializing param=model.layers.19.self_attn.q_norm.weight] Loading weights: 71%|███████ | 221/311 [00:01<00:00, 172.13it/s, Materializing param=model.layers.19.self_attn.q_proj.weight] Loading weights: 71%|███████ | 221/311 [00:01<00:00, 172.13it/s, Materializing param=model.layers.19.self_attn.q_proj.weight] Loading weights: 71%|███████▏ | 222/311 [00:01<00:00, 172.13it/s, Materializing param=model.layers.19.self_attn.v_proj.weight] Loading weights: 71%|███████▏ | 222/311 [00:01<00:00, 172.13it/s, Materializing param=model.layers.19.self_attn.v_proj.weight] Loading weights: 72%|███████▏ | 223/311 [00:01<00:00, 172.13it/s, Materializing param=model.layers.20.input_layernorm.weight] Loading weights: 72%|███████▏ | 223/311 [00:01<00:00, 172.13it/s, Materializing param=model.layers.20.input_layernorm.weight] Loading weights: 72%|███████▏ | 224/311 [00:01<00:00, 172.13it/s, Materializing param=model.layers.20.mlp.down_proj.weight] Loading weights: 72%|███████▏ | 224/311 [00:01<00:00, 172.13it/s, Materializing param=model.layers.20.mlp.down_proj.weight] Loading weights: 72%|███████▏ | 225/311 [00:01<00:00, 172.13it/s, Materializing param=model.layers.20.mlp.gate_proj.weight] Loading weights: 72%|███████▏ | 225/311 [00:01<00:00, 172.13it/s, Materializing param=model.layers.20.mlp.gate_proj.weight] Loading weights: 73%|███████▎ | 226/311 [00:01<00:00, 172.13it/s, Materializing param=model.layers.20.mlp.up_proj.weight] Loading weights: 73%|███████▎ | 226/311 [00:01<00:00, 172.13it/s, Materializing param=model.layers.20.mlp.up_proj.weight] Loading weights: 73%|███████▎ | 227/311 [00:01<00:00, 172.13it/s, Materializing param=model.layers.20.post_attention_layernorm.weight] Loading weights: 73%|███████▎ | 227/311 [00:01<00:00, 172.13it/s, Materializing param=model.layers.20.post_attention_layernorm.weight] Loading weights: 73%|███████▎ | 228/311 [00:01<00:00, 172.13it/s, Materializing param=model.layers.20.self_attn.k_norm.weight] Loading weights: 73%|███████▎ | 228/311 [00:01<00:00, 172.13it/s, Materializing param=model.layers.20.self_attn.k_norm.weight] Loading weights: 74%|███████▎ | 229/311 [00:01<00:00, 172.13it/s, Materializing param=model.layers.20.self_attn.k_proj.weight] Loading weights: 74%|███████▎ | 229/311 [00:01<00:00, 172.13it/s, Materializing param=model.layers.20.self_attn.k_proj.weight] Loading weights: 74%|███████▍ | 230/311 [00:01<00:00, 172.13it/s, Materializing param=model.layers.20.self_attn.o_proj.weight] Loading weights: 74%|███████▍ | 230/311 [00:01<00:00, 172.13it/s, Materializing param=model.layers.20.self_attn.o_proj.weight] Loading weights: 74%|███████▍ | 231/311 [00:01<00:00, 172.13it/s, Materializing param=model.layers.20.self_attn.q_norm.weight] Loading weights: 74%|███████▍ | 231/311 [00:01<00:00, 172.13it/s, Materializing param=model.layers.20.self_attn.q_norm.weight] Loading weights: 75%|███████▍ | 232/311 [00:01<00:00, 172.13it/s, Materializing param=model.layers.20.self_attn.q_proj.weight] Loading weights: 75%|███████▍ | 232/311 [00:01<00:00, 172.13it/s, Materializing param=model.layers.20.self_attn.q_proj.weight] Loading weights: 75%|███████▍ | 233/311 [00:01<00:00, 172.13it/s, Materializing param=model.layers.20.self_attn.v_proj.weight] Loading weights: 75%|███████▍ | 233/311 [00:01<00:00, 172.13it/s, Materializing param=model.layers.20.self_attn.v_proj.weight] Loading weights: 75%|███████▌ | 234/311 [00:01<00:00, 172.13it/s, Materializing param=model.layers.21.input_layernorm.weight] Loading weights: 75%|███████▌ | 234/311 [00:01<00:00, 172.13it/s, Materializing param=model.layers.21.input_layernorm.weight] Loading weights: 76%|███████▌ | 235/311 [00:01<00:00, 172.13it/s, Materializing param=model.layers.21.mlp.down_proj.weight] Loading weights: 76%|███████▌ | 235/311 [00:01<00:00, 172.13it/s, Materializing param=model.layers.21.mlp.down_proj.weight] Loading weights: 76%|███████▌ | 236/311 [00:01<00:00, 172.13it/s, Materializing param=model.layers.21.mlp.gate_proj.weight] Loading weights: 76%|███████▌ | 236/311 [00:01<00:00, 172.13it/s, Materializing param=model.layers.21.mlp.gate_proj.weight] Loading weights: 76%|███████▌ | 237/311 [00:01<00:00, 172.13it/s, Materializing param=model.layers.21.mlp.up_proj.weight] Loading weights: 76%|███████▌ | 237/311 [00:01<00:00, 172.13it/s, Materializing param=model.layers.21.mlp.up_proj.weight] Loading weights: 77%|███████▋ | 238/311 [00:01<00:00, 172.13it/s, Materializing param=model.layers.21.post_attention_layernorm.weight] Loading weights: 77%|███████▋ | 238/311 [00:01<00:00, 172.13it/s, Materializing param=model.layers.21.post_attention_layernorm.weight] Loading weights: 77%|███████▋ | 239/311 [00:01<00:00, 172.13it/s, Materializing param=model.layers.21.self_attn.k_norm.weight] Loading weights: 77%|███████▋ | 239/311 [00:01<00:00, 172.13it/s, Materializing param=model.layers.21.self_attn.k_norm.weight] Loading weights: 77%|███████▋ | 240/311 [00:01<00:00, 172.13it/s, Materializing param=model.layers.21.self_attn.k_proj.weight] Loading weights: 77%|███████▋ | 240/311 [00:01<00:00, 172.13it/s, Materializing param=model.layers.21.self_attn.k_proj.weight] Loading weights: 77%|███████▋ | 241/311 [00:01<00:00, 172.13it/s, Materializing param=model.layers.21.self_attn.o_proj.weight] Loading weights: 77%|███████▋ | 241/311 [00:01<00:00, 172.13it/s, Materializing param=model.layers.21.self_attn.o_proj.weight] Loading weights: 78%|███████▊ | 242/311 [00:01<00:00, 172.13it/s, Materializing param=model.layers.21.self_attn.q_norm.weight] Loading weights: 78%|███████▊ | 242/311 [00:01<00:00, 172.13it/s, Materializing param=model.layers.21.self_attn.q_norm.weight] Loading weights: 78%|███████▊ | 243/311 [00:01<00:00, 172.13it/s, Materializing param=model.layers.21.self_attn.q_proj.weight] Loading weights: 78%|███████▊ | 243/311 [00:01<00:00, 172.13it/s, Materializing param=model.layers.21.self_attn.q_proj.weight] Loading weights: 78%|███████▊ | 244/311 [00:01<00:00, 172.13it/s, Materializing param=model.layers.21.self_attn.v_proj.weight] Loading weights: 78%|███████▊ | 244/311 [00:01<00:00, 172.13it/s, Materializing param=model.layers.21.self_attn.v_proj.weight] Loading weights: 79%|███████▉ | 245/311 [00:01<00:00, 172.13it/s, Materializing param=model.layers.22.input_layernorm.weight] Loading weights: 79%|███████▉ | 245/311 [00:01<00:00, 172.13it/s, Materializing param=model.layers.22.input_layernorm.weight] Loading weights: 79%|███████▉ | 246/311 [00:01<00:00, 172.13it/s, Materializing param=model.layers.22.mlp.down_proj.weight] Loading weights: 79%|███████▉ | 246/311 [00:01<00:00, 172.13it/s, Materializing param=model.layers.22.mlp.down_proj.weight] Loading weights: 79%|███████▉ | 247/311 [00:01<00:00, 172.13it/s, Materializing param=model.layers.22.mlp.gate_proj.weight] Loading weights: 79%|███████▉ | 247/311 [00:01<00:00, 172.13it/s, Materializing param=model.layers.22.mlp.gate_proj.weight] Loading weights: 80%|███████▉ | 248/311 [00:01<00:00, 172.13it/s, Materializing param=model.layers.22.mlp.up_proj.weight] Loading weights: 80%|███████▉ | 248/311 [00:01<00:00, 172.13it/s, Materializing param=model.layers.22.mlp.up_proj.weight] Loading weights: 80%|████████ | 249/311 [00:01<00:00, 172.13it/s, Materializing param=model.layers.22.post_attention_layernorm.weight] Loading weights: 80%|████████ | 249/311 [00:01<00:00, 172.13it/s, Materializing param=model.layers.22.post_attention_layernorm.weight] Loading weights: 80%|████████ | 250/311 [00:01<00:00, 172.13it/s, Materializing param=model.layers.22.self_attn.k_norm.weight] Loading weights: 80%|████████ | 250/311 [00:01<00:00, 172.13it/s, Materializing param=model.layers.22.self_attn.k_norm.weight] Loading weights: 81%|████████ | 251/311 [00:01<00:00, 172.13it/s, Materializing param=model.layers.22.self_attn.k_proj.weight] Loading weights: 81%|████████ | 251/311 [00:01<00:00, 172.13it/s, Materializing param=model.layers.22.self_attn.k_proj.weight] Loading weights: 81%|████████ | 252/311 [00:01<00:00, 172.13it/s, Materializing param=model.layers.22.self_attn.o_proj.weight] Loading weights: 81%|████████ | 252/311 [00:01<00:00, 172.13it/s, Materializing param=model.layers.22.self_attn.o_proj.weight] Loading weights: 81%|████████▏ | 253/311 [00:01<00:00, 172.13it/s, Materializing param=model.layers.22.self_attn.q_norm.weight] Loading weights: 81%|████████▏ | 253/311 [00:01<00:00, 172.13it/s, Materializing param=model.layers.22.self_attn.q_norm.weight] Loading weights: 82%|████████▏ | 254/311 [00:01<00:00, 172.13it/s, Materializing param=model.layers.22.self_attn.q_proj.weight] Loading weights: 82%|████████▏ | 254/311 [00:01<00:00, 172.13it/s, Materializing param=model.layers.22.self_attn.q_proj.weight] Loading weights: 82%|████████▏ | 255/311 [00:01<00:00, 172.13it/s, Materializing param=model.layers.22.self_attn.v_proj.weight] Loading weights: 82%|████████▏ | 255/311 [00:01<00:00, 172.13it/s, Materializing param=model.layers.22.self_attn.v_proj.weight] Loading weights: 82%|████████▏ | 256/311 [00:01<00:00, 172.13it/s, Materializing param=model.layers.23.input_layernorm.weight] Loading weights: 82%|████████▏ | 256/311 [00:01<00:00, 172.13it/s, Materializing param=model.layers.23.input_layernorm.weight] Loading weights: 83%|████████▎ | 257/311 [00:01<00:00, 172.13it/s, Materializing param=model.layers.23.mlp.down_proj.weight] Loading weights: 83%|████████▎ | 257/311 [00:01<00:00, 172.13it/s, Materializing param=model.layers.23.mlp.down_proj.weight] Loading weights: 83%|████████▎ | 258/311 [00:01<00:00, 172.13it/s, Materializing param=model.layers.23.mlp.gate_proj.weight] Loading weights: 83%|████████▎ | 258/311 [00:01<00:00, 172.13it/s, Materializing param=model.layers.23.mlp.gate_proj.weight] Loading weights: 83%|████████▎ | 259/311 [00:01<00:00, 172.13it/s, Materializing param=model.layers.23.mlp.up_proj.weight] Loading weights: 83%|████████▎ | 259/311 [00:01<00:00, 172.13it/s, Materializing param=model.layers.23.mlp.up_proj.weight] Loading weights: 84%|████████▎ | 260/311 [00:01<00:00, 172.13it/s, Materializing param=model.layers.23.post_attention_layernorm.weight] Loading weights: 84%|████████▎ | 260/311 [00:01<00:00, 172.13it/s, Materializing param=model.layers.23.post_attention_layernorm.weight] Loading weights: 84%|████████▍ | 261/311 [00:01<00:00, 172.13it/s, Materializing param=model.layers.23.self_attn.k_norm.weight] Loading weights: 84%|████████▍ | 261/311 [00:01<00:00, 172.13it/s, Materializing param=model.layers.23.self_attn.k_norm.weight] Loading weights: 84%|████████▍ | 262/311 [00:01<00:00, 172.13it/s, Materializing param=model.layers.23.self_attn.k_proj.weight] Loading weights: 84%|████████▍ | 262/311 [00:01<00:00, 172.13it/s, Materializing param=model.layers.23.self_attn.k_proj.weight] Loading weights: 85%|████████▍ | 263/311 [00:01<00:00, 172.13it/s, Materializing param=model.layers.23.self_attn.o_proj.weight] Loading weights: 85%|████████▍ | 263/311 [00:01<00:00, 172.13it/s, Materializing param=model.layers.23.self_attn.o_proj.weight] Loading weights: 85%|████████▍ | 264/311 [00:01<00:00, 172.13it/s, Materializing param=model.layers.23.self_attn.q_norm.weight] Loading weights: 85%|████████▍ | 264/311 [00:01<00:00, 172.13it/s, Materializing param=model.layers.23.self_attn.q_norm.weight] Loading weights: 85%|████████▌ | 265/311 [00:01<00:00, 172.13it/s, Materializing param=model.layers.23.self_attn.q_proj.weight] Loading weights: 85%|████████▌ | 265/311 [00:01<00:00, 172.13it/s, Materializing param=model.layers.23.self_attn.q_proj.weight] Loading weights: 86%|████████▌ | 266/311 [00:01<00:00, 172.13it/s, Materializing param=model.layers.23.self_attn.v_proj.weight] Loading weights: 86%|████████▌ | 266/311 [00:01<00:00, 172.13it/s, Materializing param=model.layers.23.self_attn.v_proj.weight] Loading weights: 86%|████████▌ | 267/311 [00:01<00:00, 172.13it/s, Materializing param=model.layers.24.input_layernorm.weight] Loading weights: 86%|████████▌ | 267/311 [00:01<00:00, 172.13it/s, Materializing param=model.layers.24.input_layernorm.weight] Loading weights: 86%|████████▌ | 268/311 [00:01<00:00, 172.13it/s, Materializing param=model.layers.24.mlp.down_proj.weight] Loading weights: 86%|████████▌ | 268/311 [00:01<00:00, 172.13it/s, Materializing param=model.layers.24.mlp.down_proj.weight] Loading weights: 86%|████████▋ | 269/311 [00:01<00:00, 172.13it/s, Materializing param=model.layers.24.mlp.gate_proj.weight] Loading weights: 86%|████████▋ | 269/311 [00:01<00:00, 172.13it/s, Materializing param=model.layers.24.mlp.gate_proj.weight] Loading weights: 87%|████████▋ | 270/311 [00:01<00:00, 172.13it/s, Materializing param=model.layers.24.mlp.up_proj.weight] Loading weights: 87%|████████▋ | 270/311 [00:01<00:00, 172.13it/s, Materializing param=model.layers.24.mlp.up_proj.weight] Loading weights: 87%|████████▋ | 271/311 [00:01<00:00, 172.13it/s, Materializing param=model.layers.24.post_attention_layernorm.weight] Loading weights: 87%|████████▋ | 271/311 [00:01<00:00, 172.13it/s, Materializing param=model.layers.24.post_attention_layernorm.weight] Loading weights: 87%|████████▋ | 272/311 [00:01<00:00, 172.13it/s, Materializing param=model.layers.24.self_attn.k_norm.weight] Loading weights: 87%|████████▋ | 272/311 [00:01<00:00, 172.13it/s, Materializing param=model.layers.24.self_attn.k_norm.weight] Loading weights: 88%|████████▊ | 273/311 [00:01<00:00, 172.13it/s, Materializing param=model.layers.24.self_attn.k_proj.weight] Loading weights: 88%|████████▊ | 273/311 [00:01<00:00, 172.13it/s, Materializing param=model.layers.24.self_attn.k_proj.weight] Loading weights: 88%|████████▊ | 274/311 [00:01<00:00, 172.13it/s, Materializing param=model.layers.24.self_attn.o_proj.weight] Loading weights: 88%|████████▊ | 274/311 [00:01<00:00, 172.13it/s, Materializing param=model.layers.24.self_attn.o_proj.weight] Loading weights: 88%|████████▊ | 275/311 [00:01<00:00, 172.13it/s, Materializing param=model.layers.24.self_attn.q_norm.weight] Loading weights: 88%|████████▊ | 275/311 [00:01<00:00, 172.13it/s, Materializing param=model.layers.24.self_attn.q_norm.weight] Loading weights: 89%|████████▊ | 276/311 [00:01<00:00, 172.13it/s, Materializing param=model.layers.24.self_attn.q_proj.weight] Loading weights: 89%|████████▊ | 276/311 [00:01<00:00, 172.13it/s, Materializing param=model.layers.24.self_attn.q_proj.weight] Loading weights: 89%|████████▉ | 277/311 [00:01<00:00, 172.13it/s, Materializing param=model.layers.24.self_attn.v_proj.weight] Loading weights: 89%|████████▉ | 277/311 [00:01<00:00, 172.13it/s, Materializing param=model.layers.24.self_attn.v_proj.weight] Loading weights: 89%|████████▉ | 278/311 [00:01<00:00, 172.13it/s, Materializing param=model.layers.25.input_layernorm.weight] Loading weights: 89%|████████▉ | 278/311 [00:01<00:00, 172.13it/s, Materializing param=model.layers.25.input_layernorm.weight] Loading weights: 90%|████████▉ | 279/311 [00:01<00:00, 172.13it/s, Materializing param=model.layers.25.mlp.down_proj.weight] Loading weights: 90%|████████▉ | 279/311 [00:01<00:00, 172.13it/s, Materializing param=model.layers.25.mlp.down_proj.weight] Loading weights: 90%|█████████ | 280/311 [00:01<00:00, 172.13it/s, Materializing param=model.layers.25.mlp.gate_proj.weight] Loading weights: 90%|█████████ | 280/311 [00:01<00:00, 172.13it/s, Materializing param=model.layers.25.mlp.gate_proj.weight] Loading weights: 90%|█████████ | 281/311 [00:01<00:00, 172.13it/s, Materializing param=model.layers.25.mlp.up_proj.weight] Loading weights: 90%|█████████ | 281/311 [00:01<00:00, 172.13it/s, Materializing param=model.layers.25.mlp.up_proj.weight] Loading weights: 91%|█████████ | 282/311 [00:01<00:00, 172.13it/s, Materializing param=model.layers.25.post_attention_layernorm.weight] Loading weights: 91%|█████████ | 282/311 [00:01<00:00, 172.13it/s, Materializing param=model.layers.25.post_attention_layernorm.weight] Loading weights: 91%|█████████ | 283/311 [00:01<00:00, 172.13it/s, Materializing param=model.layers.25.self_attn.k_norm.weight] Loading weights: 91%|█████████ | 283/311 [00:01<00:00, 172.13it/s, Materializing param=model.layers.25.self_attn.k_norm.weight] Loading weights: 91%|█████████▏| 284/311 [00:01<00:00, 172.13it/s, Materializing param=model.layers.25.self_attn.k_proj.weight] Loading weights: 91%|█████████▏| 284/311 [00:01<00:00, 172.13it/s, Materializing param=model.layers.25.self_attn.k_proj.weight] Loading weights: 92%|█████████▏| 285/311 [00:01<00:00, 172.13it/s, Materializing param=model.layers.25.self_attn.o_proj.weight] Loading weights: 92%|█████████▏| 285/311 [00:01<00:00, 172.13it/s, Materializing param=model.layers.25.self_attn.o_proj.weight] Loading weights: 92%|█████████▏| 286/311 [00:01<00:00, 172.13it/s, Materializing param=model.layers.25.self_attn.q_norm.weight] Loading weights: 92%|█████████▏| 286/311 [00:01<00:00, 172.13it/s, Materializing param=model.layers.25.self_attn.q_norm.weight] Loading weights: 92%|█████████▏| 287/311 [00:01<00:00, 172.13it/s, Materializing param=model.layers.25.self_attn.q_proj.weight] Loading weights: 92%|█████████▏| 287/311 [00:01<00:00, 172.13it/s, Materializing param=model.layers.25.self_attn.q_proj.weight] Loading weights: 93%|█████████▎| 288/311 [00:01<00:00, 172.13it/s, Materializing param=model.layers.25.self_attn.v_proj.weight] Loading weights: 93%|█████████▎| 288/311 [00:01<00:00, 172.13it/s, Materializing param=model.layers.25.self_attn.v_proj.weight] Loading weights: 93%|█████████▎| 289/311 [00:01<00:00, 172.13it/s, Materializing param=model.layers.26.input_layernorm.weight] Loading weights: 93%|█████████▎| 289/311 [00:01<00:00, 172.13it/s, Materializing param=model.layers.26.input_layernorm.weight] Loading weights: 93%|█████████▎| 290/311 [00:01<00:00, 172.13it/s, Materializing param=model.layers.26.mlp.down_proj.weight] Loading weights: 93%|█████████▎| 290/311 [00:01<00:00, 172.13it/s, Materializing param=model.layers.26.mlp.down_proj.weight] Loading weights: 94%|█████████▎| 291/311 [00:01<00:00, 172.13it/s, Materializing param=model.layers.26.mlp.gate_proj.weight] Loading weights: 94%|█████████▎| 291/311 [00:01<00:00, 172.13it/s, Materializing param=model.layers.26.mlp.gate_proj.weight] Loading weights: 94%|█████████▍| 292/311 [00:01<00:00, 172.13it/s, Materializing param=model.layers.26.mlp.up_proj.weight] Loading weights: 94%|█████████▍| 292/311 [00:01<00:00, 172.13it/s, Materializing param=model.layers.26.mlp.up_proj.weight] Loading weights: 94%|█████████▍| 293/311 [00:01<00:00, 172.13it/s, Materializing param=model.layers.26.post_attention_layernorm.weight] Loading weights: 94%|█████████▍| 293/311 [00:01<00:00, 172.13it/s, Materializing param=model.layers.26.post_attention_layernorm.weight] Loading weights: 95%|█████████▍| 294/311 [00:01<00:00, 172.13it/s, Materializing param=model.layers.26.self_attn.k_norm.weight] Loading weights: 95%|█████████▍| 294/311 [00:01<00:00, 172.13it/s, Materializing param=model.layers.26.self_attn.k_norm.weight] Loading weights: 95%|█████████▍| 295/311 [00:01<00:00, 172.13it/s, Materializing param=model.layers.26.self_attn.k_proj.weight] Loading weights: 95%|█████████▍| 295/311 [00:01<00:00, 172.13it/s, Materializing param=model.layers.26.self_attn.k_proj.weight] Loading weights: 95%|█████████▌| 296/311 [00:01<00:00, 172.13it/s, Materializing param=model.layers.26.self_attn.o_proj.weight] Loading weights: 95%|█████████▌| 296/311 [00:01<00:00, 172.13it/s, Materializing param=model.layers.26.self_attn.o_proj.weight] Loading weights: 95%|█████████▌| 297/311 [00:01<00:00, 374.43it/s, Materializing param=model.layers.26.self_attn.o_proj.weight] Loading weights: 95%|█████████▌| 297/311 [00:01<00:00, 374.43it/s, Materializing param=model.layers.26.self_attn.q_norm.weight] Loading weights: 95%|█████████▌| 297/311 [00:01<00:00, 374.43it/s, Materializing param=model.layers.26.self_attn.q_norm.weight] Loading weights: 96%|█████████▌| 298/311 [00:01<00:00, 374.43it/s, Materializing param=model.layers.26.self_attn.q_proj.weight] Loading weights: 96%|█████████▌| 298/311 [00:01<00:00, 374.43it/s, Materializing param=model.layers.26.self_attn.q_proj.weight] Loading weights: 96%|█████████▌| 299/311 [00:01<00:00, 374.43it/s, Materializing param=model.layers.26.self_attn.v_proj.weight] Loading weights: 96%|█████████▌| 299/311 [00:01<00:00, 374.43it/s, Materializing param=model.layers.26.self_attn.v_proj.weight] Loading weights: 96%|█████████▋| 300/311 [00:01<00:00, 374.43it/s, Materializing param=model.layers.27.input_layernorm.weight] Loading weights: 96%|█████████▋| 300/311 [00:01<00:00, 374.43it/s, Materializing param=model.layers.27.input_layernorm.weight] Loading weights: 97%|█████████▋| 301/311 [00:01<00:00, 374.43it/s, Materializing param=model.layers.27.mlp.down_proj.weight] Loading weights: 97%|█████████▋| 301/311 [00:01<00:00, 374.43it/s, Materializing param=model.layers.27.mlp.down_proj.weight] Loading weights: 97%|█████████▋| 302/311 [00:01<00:00, 374.43it/s, Materializing param=model.layers.27.mlp.gate_proj.weight] Loading weights: 97%|█████████▋| 302/311 [00:01<00:00, 374.43it/s, Materializing param=model.layers.27.mlp.gate_proj.weight] Loading weights: 97%|█████████▋| 303/311 [00:01<00:00, 374.43it/s, Materializing param=model.layers.27.mlp.up_proj.weight] Loading weights: 97%|█████████▋| 303/311 [00:01<00:00, 374.43it/s, Materializing param=model.layers.27.mlp.up_proj.weight] Loading weights: 98%|█████████▊| 304/311 [00:01<00:00, 374.43it/s, Materializing param=model.layers.27.post_attention_layernorm.weight] Loading weights: 98%|█████████▊| 304/311 [00:01<00:00, 374.43it/s, Materializing param=model.layers.27.post_attention_layernorm.weight] Loading weights: 98%|█████████▊| 305/311 [00:01<00:00, 374.43it/s, Materializing param=model.layers.27.self_attn.k_norm.weight] Loading weights: 98%|█████████▊| 305/311 [00:01<00:00, 374.43it/s, Materializing param=model.layers.27.self_attn.k_norm.weight] Loading weights: 98%|█████████▊| 306/311 [00:01<00:00, 374.43it/s, Materializing param=model.layers.27.self_attn.k_proj.weight] Loading weights: 98%|█████████▊| 306/311 [00:01<00:00, 374.43it/s, Materializing param=model.layers.27.self_attn.k_proj.weight] Loading weights: 99%|█████████▊| 307/311 [00:01<00:00, 374.43it/s, Materializing param=model.layers.27.self_attn.o_proj.weight] Loading weights: 99%|█████████▊| 307/311 [00:01<00:00, 374.43it/s, Materializing param=model.layers.27.self_attn.o_proj.weight] Loading weights: 99%|█████████▉| 308/311 [00:01<00:00, 374.43it/s, Materializing param=model.layers.27.self_attn.q_norm.weight] Loading weights: 99%|█████████▉| 308/311 [00:01<00:00, 374.43it/s, Materializing param=model.layers.27.self_attn.q_norm.weight] Loading weights: 99%|█████████▉| 309/311 [00:01<00:00, 374.43it/s, Materializing param=model.layers.27.self_attn.q_proj.weight] Loading weights: 99%|█████████▉| 309/311 [00:01<00:00, 374.43it/s, Materializing param=model.layers.27.self_attn.q_proj.weight] Loading weights: 100%|█████████▉| 310/311 [00:01<00:00, 374.43it/s, Materializing param=model.layers.27.self_attn.v_proj.weight] Loading weights: 100%|█████████▉| 310/311 [00:01<00:00, 374.43it/s, Materializing param=model.layers.27.self_attn.v_proj.weight] Loading weights: 100%|██████████| 311/311 [00:01<00:00, 374.43it/s, Materializing param=model.norm.weight] Loading weights: 100%|██████████| 311/311 [00:01<00:00, 374.43it/s, Materializing param=model.norm.weight] Loading weights: 100%|██████████| 311/311 [00:01<00:00, 167.36it/s, Materializing param=model.norm.weight] +The tied weights mapping and config for this model specifies to tie model.embed_tokens.weight to lm_head.weight, but both are present in the checkpoints, so we will NOT tie them. You should update the config with `tie_word_embeddings=False` to silence this warning +The tied weights mapping and config for this model specifies to tie model.embed_tokens.weight to lm_head.weight, but both are present in the checkpoints, so we will NOT tie them. You should update the config with `tie_word_embeddings=False` to silence this warning +The tied weights mapping and config for this model specifies to tie model.embed_tokens.weight to lm_head.weight, but both are present in the checkpoints, so we will NOT tie them. You should update the config with `tie_word_embeddings=False` to silence this warning +The tied weights mapping and config for this model specifies to tie model.embed_tokens.weight to lm_head.weight, but both are present in the checkpoints, so we will NOT tie them. You should update the config with `tie_word_embeddings=False` to silence this warning +The tied weights mapping and config for this model specifies to tie model.embed_tokens.weight to lm_head.weight, but both are present in the checkpoints, so we will NOT tie them. You should update the config with `tie_word_embeddings=False` to silence this warning +[INFO|configuration_utils.py:967] 2026-04-29 15:44:24,477 >> loading configuration file generation_config.json from cache at /root/.cache/huggingface/hub/models--Qwen--Qwen3-1.7B/snapshots/70d244cc86ccca08cf5af4e1e306ecf908b1ad5e/generation_config.json +The tied weights mapping and config for this model specifies to tie model.embed_tokens.weight to lm_head.weight, but both are present in the checkpoints, so we will NOT tie them. You should update the config with `tie_word_embeddings=False` to silence this warning +[INFO|configuration_utils.py:1014] 2026-04-29 15:44:24,477 >> Generate config GenerationConfig { + "bos_token_id": 151643, + "do_sample": true, + "eos_token_id": [ + 151645, + 151643 + ], + "pad_token_id": 151643, + "temperature": 0.6, + "top_k": 20, + "top_p": 0.95 +} + +The tied weights mapping and config for this model specifies to tie model.embed_tokens.weight to lm_head.weight, but both are present in the checkpoints, so we will NOT tie them. You should update the config with `tie_word_embeddings=False` to silence this warning +[INFO|dynamic_module_utils.py:406] 2026-04-29 15:44:24,593 >> Could not locate the custom_generate/generate.py inside Qwen/Qwen3-1.7B. +[INFO|2026-04-29 15:44:24] llamafactory.model.model_utils.checkpointing:144 >> Gradient checkpointing enabled. +[INFO|2026-04-29 15:44:24] llamafactory.model.model_utils.attention:144 >> Using torch SDPA for faster training and inference. +[INFO|2026-04-29 15:44:24] llamafactory.model.adapter:144 >> Upcasting trainable params to float32. +[INFO|2026-04-29 15:44:24] llamafactory.model.adapter:144 >> Fine-tuning method: Full +[INFO|2026-04-29 15:44:24] llamafactory.model.loader:144 >> trainable params: 2,031,739,904 || all params: 2,031,739,904 || trainable%: 100.0000 +[WARNING|trainer_utils.py:1234] 2026-04-29 15:44:24,831 >> The tokenizer has new PAD/BOS/EOS tokens that differ from the model config and generation config. The model config and generation config were aligned accordingly, being updated with the tokenizer's values. Updated tokens: {'bos_token_id': None, 'pad_token_id': 151643}. +The tokenizer has new PAD/BOS/EOS tokens that differ from the model config and generation config. The model config and generation config were aligned accordingly, being updated with the tokenizer's values. Updated tokens: {'bos_token_id': None, 'pad_token_id': 151643}. +The tokenizer has new PAD/BOS/EOS tokens that differ from the model config and generation config. The model config and generation config were aligned accordingly, being updated with the tokenizer's values. Updated tokens: {'bos_token_id': None, 'pad_token_id': 151643}. +The tokenizer has new PAD/BOS/EOS tokens that differ from the model config and generation config. The model config and generation config were aligned accordingly, being updated with the tokenizer's values. Updated tokens: {'bos_token_id': None, 'pad_token_id': 151643}. +The tokenizer has new PAD/BOS/EOS tokens that differ from the model config and generation config. The model config and generation config were aligned accordingly, being updated with the tokenizer's values. Updated tokens: {'bos_token_id': None, 'pad_token_id': 151643}. +The tokenizer has new PAD/BOS/EOS tokens that differ from the model config and generation config. The model config and generation config were aligned accordingly, being updated with the tokenizer's values. Updated tokens: {'bos_token_id': None, 'pad_token_id': 151643}. +The tokenizer has new PAD/BOS/EOS tokens that differ from the model config and generation config. The model config and generation config were aligned accordingly, being updated with the tokenizer's values. Updated tokens: {'bos_token_id': None, 'pad_token_id': 151643}. +The tokenizer has new PAD/BOS/EOS tokens that differ from the model config and generation config. The model config and generation config were aligned accordingly, being updated with the tokenizer's values. Updated tokens: {'bos_token_id': None, 'pad_token_id': 151643}. +ywang29-p4d-debug-worker-0:2470953:2470953 [5] NCCL INFO Comm config Blocking set to 1 +ywang29-p4d-debug-worker-0:2470948:2470948 [1] NCCL INFO Comm config Blocking set to 1 +ywang29-p4d-debug-worker-0:2470954:2470954 [6] NCCL INFO Comm config Blocking set to 1 +ywang29-p4d-debug-worker-0:2470947:2470947 [0] NCCL INFO Comm config Blocking set to 1 +ywang29-p4d-debug-worker-0:2470952:2470952 [4] NCCL INFO Comm config Blocking set to 1 +ywang29-p4d-debug-worker-0:2470955:2470955 [7] NCCL INFO Comm config Blocking set to 1 +ywang29-p4d-debug-worker-0:2470951:2470951 [3] NCCL INFO Comm config Blocking set to 1 +ywang29-p4d-debug-worker-0:2470949:2470949 [2] NCCL INFO Comm config Blocking set to 1 +ywang29-p4d-debug-worker-0:2470949:2470949 [2] NCCL INFO Initialized NET plugin Socket +ywang29-p4d-debug-worker-0:2470951:2470951 [3] NCCL INFO Initialized NET plugin Socket +ywang29-p4d-debug-worker-0:2470952:2470952 [4] NCCL INFO Initialized NET plugin Socket +ywang29-p4d-debug-worker-0:2470948:2470948 [1] NCCL INFO Initialized NET plugin Socket +ywang29-p4d-debug-worker-0:2470949:2470949 [2] NCCL INFO Assigned NET plugin Socket to comm +ywang29-p4d-debug-worker-0:2470949:2470949 [2] NCCL INFO Using network Socket +ywang29-p4d-debug-worker-0:2470955:2470955 [7] NCCL INFO Initialized NET plugin Socket +ywang29-p4d-debug-worker-0:2470953:2470953 [5] NCCL INFO Initialized NET plugin Socket +ywang29-p4d-debug-worker-0:2470952:2470952 [4] NCCL INFO Assigned NET plugin Socket to comm +ywang29-p4d-debug-worker-0:2470954:2470954 [6] NCCL INFO Initialized NET plugin Socket +ywang29-p4d-debug-worker-0:2470947:2470947 [0] NCCL INFO Initialized NET plugin Socket +ywang29-p4d-debug-worker-0:2470948:2470948 [1] NCCL INFO Assigned NET plugin Socket to comm +ywang29-p4d-debug-worker-0:2470952:2470952 [4] NCCL INFO Using network Socket +ywang29-p4d-debug-worker-0:2470948:2470948 [1] NCCL INFO Using network Socket +ywang29-p4d-debug-worker-0:2470951:2470951 [3] NCCL INFO Assigned NET plugin Socket to comm +ywang29-p4d-debug-worker-0:2470951:2470951 [3] NCCL INFO Using network Socket +ywang29-p4d-debug-worker-0:2470955:2470955 [7] NCCL INFO Assigned NET plugin Socket to comm +ywang29-p4d-debug-worker-0:2470955:2470955 [7] NCCL INFO Using network Socket +ywang29-p4d-debug-worker-0:2470953:2470953 [5] NCCL INFO Assigned NET plugin Socket to comm +ywang29-p4d-debug-worker-0:2470953:2470953 [5] NCCL INFO Using network Socket +ywang29-p4d-debug-worker-0:2470947:2470947 [0] NCCL INFO Assigned NET plugin Socket to comm +ywang29-p4d-debug-worker-0:2470947:2470947 [0] NCCL INFO Using network Socket +ywang29-p4d-debug-worker-0:2470954:2470954 [6] NCCL INFO Assigned NET plugin Socket to comm +ywang29-p4d-debug-worker-0:2470954:2470954 [6] NCCL INFO Using network Socket +ywang29-p4d-debug-worker-0:2470949:2470949 [2] NCCL INFO ncclCommSplit comm 0x56028de3b5a0 rank 2 nranks 8 cudaDev 2 nvmlDev 2 busId 201c0 parent 0x5602809d0750 splitCount 1 color 1266629538 key 2- Init START +ywang29-p4d-debug-worker-0:2470948:2470948 [1] NCCL INFO ncclCommSplit comm 0x558d03437a10 rank 1 nranks 8 cudaDev 1 nvmlDev 1 busId 101d0 parent 0x558cf7486a10 splitCount 1 color 1266629538 key 1- Init START +ywang29-p4d-debug-worker-0:2470955:2470955 [7] NCCL INFO ncclCommSplit comm 0x55e3778f3800 rank 7 nranks 8 cudaDev 7 nvmlDev 7 busId a01d0 parent 0x55e36d363080 splitCount 1 color 1266629538 key 7- Init START +ywang29-p4d-debug-worker-0:2470953:2470953 [5] NCCL INFO ncclCommSplit comm 0x55c02c235cf0 rank 5 nranks 8 cudaDev 5 nvmlDev 5 busId 901d0 parent 0x55c0201d0a60 splitCount 1 color 1266629538 key 5- Init START +ywang29-p4d-debug-worker-0:2470951:2470951 [3] NCCL INFO ncclCommSplit comm 0x5612df7ca6e0 rank 3 nranks 8 cudaDev 3 nvmlDev 3 busId 201d0 parent 0x5612d38dcbf0 splitCount 1 color 1266629538 key 3- Init START +ywang29-p4d-debug-worker-0:2470952:2470952 [4] NCCL INFO ncclCommSplit comm 0x5647135d02c0 rank 4 nranks 8 cudaDev 4 nvmlDev 4 busId 901c0 parent 0x5647078795e0 splitCount 1 color 1266629538 key 4- Init START +ywang29-p4d-debug-worker-0:2470954:2470954 [6] NCCL INFO ncclCommSplit comm 0x560722415b60 rank 6 nranks 8 cudaDev 6 nvmlDev 6 busId a01c0 parent 0x56071a5c6320 splitCount 1 color 1266629538 key 6- Init START +ywang29-p4d-debug-worker-0:2470947:2470947 [0] NCCL INFO ncclCommSplit comm 0x55ffc26dd150 rank 0 nranks 8 cudaDev 0 nvmlDev 0 busId 101c0 parent 0x55ffbe0f7c80 splitCount 1 color 1266629538 key 0- Init START +ywang29-p4d-debug-worker-0:2470954:2470954 [6] NCCL INFO ncclTopoGetCpuAffinity: Affinity for GPU 6 is 24-47,72-95. (GPU affinity = 24-47,72-95 ; CPU affinity = 0-95). +ywang29-p4d-debug-worker-0:2470954:2470954 [6] NCCL INFO NVLS multicast support is not available on dev 6 (NVLS_NCHANNELS 0) +ywang29-p4d-debug-worker-0:2470951:2470951 [3] NCCL INFO ncclTopoGetCpuAffinity: Affinity for GPU 3 is 0-23,48-71. (GPU affinity = 0-23,48-71 ; CPU affinity = 0-95). +ywang29-p4d-debug-worker-0:2470951:2470951 [3] NCCL INFO NVLS multicast support is not available on dev 3 (NVLS_NCHANNELS 0) +ywang29-p4d-debug-worker-0:2470949:2470949 [2] NCCL INFO ncclTopoGetCpuAffinity: Affinity for GPU 2 is 0-23,48-71. (GPU affinity = 0-23,48-71 ; CPU affinity = 0-95). +ywang29-p4d-debug-worker-0:2470949:2470949 [2] NCCL INFO NVLS multicast support is not available on dev 2 (NVLS_NCHANNELS 0) +ywang29-p4d-debug-worker-0:2470948:2470948 [1] NCCL INFO ncclTopoGetCpuAffinity: Affinity for GPU 1 is 0-23,48-71. (GPU affinity = 0-23,48-71 ; CPU affinity = 0-95). +ywang29-p4d-debug-worker-0:2470948:2470948 [1] NCCL INFO NVLS multicast support is not available on dev 1 (NVLS_NCHANNELS 0) +ywang29-p4d-debug-worker-0:2470953:2470953 [5] NCCL INFO ncclTopoGetCpuAffinity: Affinity for GPU 5 is 24-47,72-95. (GPU affinity = 24-47,72-95 ; CPU affinity = 0-95). +ywang29-p4d-debug-worker-0:2470953:2470953 [5] NCCL INFO NVLS multicast support is not available on dev 5 (NVLS_NCHANNELS 0) +ywang29-p4d-debug-worker-0:2470955:2470955 [7] NCCL INFO ncclTopoGetCpuAffinity: Affinity for GPU 7 is 24-47,72-95. (GPU affinity = 24-47,72-95 ; CPU affinity = 0-95). +ywang29-p4d-debug-worker-0:2470955:2470955 [7] NCCL INFO NVLS multicast support is not available on dev 7 (NVLS_NCHANNELS 0) +ywang29-p4d-debug-worker-0:2470952:2470952 [4] NCCL INFO ncclTopoGetCpuAffinity: Affinity for GPU 4 is 24-47,72-95. (GPU affinity = 24-47,72-95 ; CPU affinity = 0-95). +ywang29-p4d-debug-worker-0:2470952:2470952 [4] NCCL INFO NVLS multicast support is not available on dev 4 (NVLS_NCHANNELS 0) +ywang29-p4d-debug-worker-0:2470947:2470947 [0] NCCL INFO ncclTopoGetCpuAffinity: Affinity for GPU 0 is 0-23,48-71. (GPU affinity = 0-23,48-71 ; CPU affinity = 0-95). +ywang29-p4d-debug-worker-0:2470947:2470947 [0] NCCL INFO NVLS multicast support is not available on dev 0 (NVLS_NCHANNELS 0) +ywang29-p4d-debug-worker-0:2470948:2470948 [1] NCCL INFO comm 0x558d03437a10 rank 1 nRanks 8 nNodes 1 localRanks 8 localRank 1 MNNVL 0 +ywang29-p4d-debug-worker-0:2470947:2470947 [0] NCCL INFO comm 0x55ffc26dd150 rank 0 nRanks 8 nNodes 1 localRanks 8 localRank 0 MNNVL 0 +ywang29-p4d-debug-worker-0:2470955:2470955 [7] NCCL INFO comm 0x55e3778f3800 rank 7 nRanks 8 nNodes 1 localRanks 8 localRank 7 MNNVL 0 +ywang29-p4d-debug-worker-0:2470954:2470954 [6] NCCL INFO comm 0x560722415b60 rank 6 nRanks 8 nNodes 1 localRanks 8 localRank 6 MNNVL 0 +ywang29-p4d-debug-worker-0:2470953:2470953 [5] NCCL INFO comm 0x55c02c235cf0 rank 5 nRanks 8 nNodes 1 localRanks 8 localRank 5 MNNVL 0 +ywang29-p4d-debug-worker-0:2470949:2470949 [2] NCCL INFO comm 0x56028de3b5a0 rank 2 nRanks 8 nNodes 1 localRanks 8 localRank 2 MNNVL 0 +ywang29-p4d-debug-worker-0:2470952:2470952 [4] NCCL INFO comm 0x5647135d02c0 rank 4 nRanks 8 nNodes 1 localRanks 8 localRank 4 MNNVL 0 +ywang29-p4d-debug-worker-0:2470947:2470947 [0] NCCL INFO Channel 00/24 : 0 1 2 3 4 5 6 7 +ywang29-p4d-debug-worker-0:2470951:2470951 [3] NCCL INFO comm 0x5612df7ca6e0 rank 3 nRanks 8 nNodes 1 localRanks 8 localRank 3 MNNVL 0 +ywang29-p4d-debug-worker-0:2470947:2470947 [0] NCCL INFO Channel 01/24 : 0 1 2 3 4 5 6 7 +ywang29-p4d-debug-worker-0:2470947:2470947 [0] NCCL INFO Channel 02/24 : 0 1 2 3 4 5 6 7 +ywang29-p4d-debug-worker-0:2470947:2470947 [0] NCCL INFO Channel 03/24 : 0 1 2 3 4 5 6 7 +ywang29-p4d-debug-worker-0:2470947:2470947 [0] NCCL INFO Channel 04/24 : 0 1 2 3 4 5 6 7 +ywang29-p4d-debug-worker-0:2470947:2470947 [0] NCCL INFO Channel 05/24 : 0 1 2 3 4 5 6 7 +ywang29-p4d-debug-worker-0:2470947:2470947 [0] NCCL INFO Channel 06/24 : 0 1 2 3 4 5 6 7 +ywang29-p4d-debug-worker-0:2470947:2470947 [0] NCCL INFO Channel 07/24 : 0 1 2 3 4 5 6 7 +ywang29-p4d-debug-worker-0:2470947:2470947 [0] NCCL INFO Channel 08/24 : 0 1 2 3 4 5 6 7 +ywang29-p4d-debug-worker-0:2470947:2470947 [0] NCCL INFO Channel 09/24 : 0 1 2 3 4 5 6 7 +ywang29-p4d-debug-worker-0:2470948:2470948 [1] NCCL INFO Trees [0] 2/-1/-1->1->0 [1] 2/-1/-1->1->0 [2] 2/-1/-1->1->0 [3] 2/-1/-1->1->0 [4] 2/-1/-1->1->0 [5] 2/-1/-1->1->0 [6] 2/-1/-1->1->0 [7] 2/-1/-1->1->0 [8] 2/-1/-1->1->0 [9] 2/-1/-1->1->0 [10] 2/-1/-1->1->0 [11] 2/-1/-1->1->0 [12] 2/-1/-1->1->0 [13] 2/-1/-1->1->0 [14] 2/-1/-1->1->0 [15] 2/-1/-1->1->0 [16] 2/-1/-1->1->0 [17] 2/-1/-1->1->0 [18] 2/-1/-1->1->0 [19] 2/-1/-1->1->0 [20] 2/-1/-1->1->0 [21] 2/-1/-1->1->0 [22] 2/-1/-1->1->0 [23] 2/-1/-1->1->0 +ywang29-p4d-debug-worker-0:2470948:2470948 [1] NCCL INFO P2P Chunksize set to 524288 +ywang29-p4d-debug-worker-0:2470947:2470947 [0] NCCL INFO Channel 10/24 : 0 1 2 3 4 5 6 7 +ywang29-p4d-debug-worker-0:2470955:2470955 [7] NCCL INFO Trees [0] -1/-1/-1->7->6 [1] -1/-1/-1->7->6 [2] -1/-1/-1->7->6 [3] -1/-1/-1->7->6 [4] -1/-1/-1->7->6 [5] -1/-1/-1->7->6 [6] -1/-1/-1->7->6 [7] -1/-1/-1->7->6 [8] -1/-1/-1->7->6 [9] -1/-1/-1->7->6 [10] -1/-1/-1->7->6 [11] -1/-1/-1->7->6 [12] -1/-1/-1->7->6 [13] -1/-1/-1->7->6 [14] -1/-1/-1->7->6 [15] -1/-1/-1->7->6 [16] -1/-1/-1->7->6 [17] -1/-1/-1->7->6 [18] -1/-1/-1->7->6 [19] -1/-1/-1->7->6 [20] -1/-1/-1->7->6 [21] -1/-1/-1->7->6 [22] -1/-1/-1->7->6 [23] -1/-1/-1->7->6 +ywang29-p4d-debug-worker-0:2470947:2470947 [0] NCCL INFO Channel 11/24 : 0 1 2 3 4 5 6 7 +ywang29-p4d-debug-worker-0:2470955:2470955 [7] NCCL INFO P2P Chunksize set to 524288 +ywang29-p4d-debug-worker-0:2470947:2470947 [0] NCCL INFO Channel 12/24 : 0 1 2 3 4 5 6 7 +ywang29-p4d-debug-worker-0:2470949:2470949 [2] NCCL INFO Trees [0] 3/-1/-1->2->1 [1] 3/-1/-1->2->1 [2] 3/-1/-1->2->1 [3] 3/-1/-1->2->1 [4] 3/-1/-1->2->1 [5] 3/-1/-1->2->1 [6] 3/-1/-1->2->1 [7] 3/-1/-1->2->1 [8] 3/-1/-1->2->1 [9] 3/-1/-1->2->1 [10] 3/-1/-1->2->1 [11] 3/-1/-1->2->1 [12] 3/-1/-1->2->1 [13] 3/-1/-1->2->1 [14] 3/-1/-1->2->1 [15] 3/-1/-1->2->1 [16] 3/-1/-1->2->1 [17] 3/-1/-1->2->1 [18] 3/-1/-1->2->1 [19] 3/-1/-1->2->1 [20] 3/-1/-1->2->1 [21] 3/-1/-1->2->1 [22] 3/-1/-1->2->1 [23] 3/-1/-1->2->1 +ywang29-p4d-debug-worker-0:2470952:2470952 [4] NCCL INFO Trees [0] 5/-1/-1->4->3 [1] 5/-1/-1->4->3 [2] 5/-1/-1->4->3 [3] 5/-1/-1->4->3 [4] 5/-1/-1->4->3 [5] 5/-1/-1->4->3 [6] 5/-1/-1->4->3 [7] 5/-1/-1->4->3 [8] 5/-1/-1->4->3 [9] 5/-1/-1->4->3 [10] 5/-1/-1->4->3 [11] 5/-1/-1->4->3 [12] 5/-1/-1->4->3 [13] 5/-1/-1->4->3 [14] 5/-1/-1->4->3 [15] 5/-1/-1->4->3 [16] 5/-1/-1->4->3 [17] 5/-1/-1->4->3 [18] 5/-1/-1->4->3 [19] 5/-1/-1->4->3 [20] 5/-1/-1->4->3 [21] 5/-1/-1->4->3 [22] 5/-1/-1->4->3 [23] 5/-1/-1->4->3 +ywang29-p4d-debug-worker-0:2470949:2470949 [2] NCCL INFO P2P Chunksize set to 524288 +ywang29-p4d-debug-worker-0:2470947:2470947 [0] NCCL INFO Channel 13/24 : 0 1 2 3 4 5 6 7 +ywang29-p4d-debug-worker-0:2470952:2470952 [4] NCCL INFO P2P Chunksize set to 524288 +ywang29-p4d-debug-worker-0:2470947:2470947 [0] NCCL INFO Channel 14/24 : 0 1 2 3 4 5 6 7 +ywang29-p4d-debug-worker-0:2470947:2470947 [0] NCCL INFO Channel 15/24 : 0 1 2 3 4 5 6 7 +ywang29-p4d-debug-worker-0:2470947:2470947 [0] NCCL INFO Channel 16/24 : 0 1 2 3 4 5 6 7 +ywang29-p4d-debug-worker-0:2470947:2470947 [0] NCCL INFO Channel 17/24 : 0 1 2 3 4 5 6 7 +ywang29-p4d-debug-worker-0:2470947:2470947 [0] NCCL INFO Channel 18/24 : 0 1 2 3 4 5 6 7 +ywang29-p4d-debug-worker-0:2470947:2470947 [0] NCCL INFO Channel 19/24 : 0 1 2 3 4 5 6 7 +ywang29-p4d-debug-worker-0:2470947:2470947 [0] NCCL INFO Channel 20/24 : 0 1 2 3 4 5 6 7 +ywang29-p4d-debug-worker-0:2470951:2470951 [3] NCCL INFO Trees [0] 4/-1/-1->3->2 [1] 4/-1/-1->3->2 [2] 4/-1/-1->3->2 [3] 4/-1/-1->3->2 [4] 4/-1/-1->3->2 [5] 4/-1/-1->3->2 [6] 4/-1/-1->3->2 [7] 4/-1/-1->3->2 [8] 4/-1/-1->3->2 [9] 4/-1/-1->3->2 [10] 4/-1/-1->3->2 [11] 4/-1/-1->3->2 [12] 4/-1/-1->3->2 [13] 4/-1/-1->3->2 [14] 4/-1/-1->3->2 [15] 4/-1/-1->3->2 [16] 4/-1/-1->3->2 [17] 4/-1/-1->3->2 [18] 4/-1/-1->3->2 [19] 4/-1/-1->3->2 [20] 4/-1/-1->3->2 [21] 4/-1/-1->3->2 [22] 4/-1/-1->3->2 [23] 4/-1/-1->3->2 +ywang29-p4d-debug-worker-0:2470947:2470947 [0] NCCL INFO Channel 21/24 : 0 1 2 3 4 5 6 7 +ywang29-p4d-debug-worker-0:2470954:2470954 [6] NCCL INFO Trees [0] 7/-1/-1->6->5 [1] 7/-1/-1->6->5 [2] 7/-1/-1->6->5 [3] 7/-1/-1->6->5 [4] 7/-1/-1->6->5 [5] 7/-1/-1->6->5 [6] 7/-1/-1->6->5 [7] 7/-1/-1->6->5 [8] 7/-1/-1->6->5 [9] 7/-1/-1->6->5 [10] 7/-1/-1->6->5 [11] 7/-1/-1->6->5 [12] 7/-1/-1->6->5 [13] 7/-1/-1->6->5 [14] 7/-1/-1->6->5 [15] 7/-1/-1->6->5 [16] 7/-1/-1->6->5 [17] 7/-1/-1->6->5 [18] 7/-1/-1->6->5 [19] 7/-1/-1->6->5 [20] 7/-1/-1->6->5 [21] 7/-1/-1->6->5 [22] 7/-1/-1->6->5 [23] 7/-1/-1->6->5 +ywang29-p4d-debug-worker-0:2470953:2470953 [5] NCCL INFO Trees [0] 6/-1/-1->5->4 [1] 6/-1/-1->5->4 [2] 6/-1/-1->5->4 [3] 6/-1/-1->5->4 [4] 6/-1/-1->5->4 [5] 6/-1/-1->5->4 [6] 6/-1/-1->5->4 [7] 6/-1/-1->5->4 [8] 6/-1/-1->5->4 [9] 6/-1/-1->5->4 [10] 6/-1/-1->5->4 [11] 6/-1/-1->5->4 [12] 6/-1/-1->5->4 [13] 6/-1/-1->5->4 [14] 6/-1/-1->5->4 [15] 6/-1/-1->5->4 [16] 6/-1/-1->5->4 [17] 6/-1/-1->5->4 [18] 6/-1/-1->5->4 [19] 6/-1/-1->5->4 [20] 6/-1/-1->5->4 [21] 6/-1/-1->5->4 [22] 6/-1/-1->5->4 [23] 6/-1/-1->5->4 +ywang29-p4d-debug-worker-0:2470951:2470951 [3] NCCL INFO P2P Chunksize set to 524288 +ywang29-p4d-debug-worker-0:2470947:2470947 [0] NCCL INFO Channel 22/24 : 0 1 2 3 4 5 6 7 +ywang29-p4d-debug-worker-0:2470954:2470954 [6] NCCL INFO P2P Chunksize set to 524288 +ywang29-p4d-debug-worker-0:2470953:2470953 [5] NCCL INFO P2P Chunksize set to 524288 +ywang29-p4d-debug-worker-0:2470947:2470947 [0] NCCL INFO Channel 23/24 : 0 1 2 3 4 5 6 7 +ywang29-p4d-debug-worker-0:2470947:2470947 [0] NCCL INFO Trees [0] 1/-1/-1->0->-1 [1] 1/-1/-1->0->-1 [2] 1/-1/-1->0->-1 [3] 1/-1/-1->0->-1 [4] 1/-1/-1->0->-1 [5] 1/-1/-1->0->-1 [6] 1/-1/-1->0->-1 [7] 1/-1/-1->0->-1 [8] 1/-1/-1->0->-1 [9] 1/-1/-1->0->-1 [10] 1/-1/-1->0->-1 [11] 1/-1/-1->0->-1 [12] 1/-1/-1->0->-1 [13] 1/-1/-1->0->-1 [14] 1/-1/-1->0->-1 [15] 1/-1/-1->0->-1 [16] 1/-1/-1->0->-1 [17] 1/-1/-1->0->-1 [18] 1/-1/-1->0->-1 [19] 1/-1/-1->0->-1 [20] 1/-1/-1->0->-1 [21] 1/-1/-1->0->-1 [22] 1/-1/-1->0->-1 [23] 1/-1/-1->0->-1 +ywang29-p4d-debug-worker-0:2470947:2470947 [0] NCCL INFO P2P Chunksize set to 524288 +ywang29-p4d-debug-worker-0:2470947:2470947 [0] NCCL INFO Check P2P Type isAllDirectP2p 1 directMode 0 isAllCudaP2p 1 +ywang29-p4d-debug-worker-0:2470947:2474553 [0] NCCL INFO [Proxy Service] Device 0 CPU core 60 +ywang29-p4d-debug-worker-0:2470947:2474554 [0] NCCL INFO [Proxy Service UDS] Device 0 CPU core 15 +ywang29-p4d-debug-worker-0:2470948:2470948 [1] NCCL INFO Check P2P Type isAllDirectP2p 1 directMode 0 isAllCudaP2p 1 +ywang29-p4d-debug-worker-0:2470948:2474555 [0] NCCL INFO [Proxy Service] Device 1 CPU core 69 +ywang29-p4d-debug-worker-0:2470948:2474556 [0] NCCL INFO [Proxy Service UDS] Device 1 CPU core 16 +ywang29-p4d-debug-worker-0:2470952:2470952 [4] NCCL INFO Check P2P Type isAllDirectP2p 1 directMode 0 isAllCudaP2p 1 +ywang29-p4d-debug-worker-0:2470951:2470951 [3] NCCL INFO Check P2P Type isAllDirectP2p 1 directMode 0 isAllCudaP2p 1 +ywang29-p4d-debug-worker-0:2470952:2474557 [0] NCCL INFO [Proxy Service] Device 4 CPU core 35 +ywang29-p4d-debug-worker-0:2470952:2474558 [0] NCCL INFO [Proxy Service UDS] Device 4 CPU core 44 +ywang29-p4d-debug-worker-0:2470951:2474559 [0] NCCL INFO [Proxy Service] Device 3 CPU core 58 +ywang29-p4d-debug-worker-0:2470951:2474560 [0] NCCL INFO [Proxy Service UDS] Device 3 CPU core 14 +ywang29-p4d-debug-worker-0:2470955:2470955 [7] NCCL INFO Check P2P Type isAllDirectP2p 1 directMode 0 isAllCudaP2p 1 +ywang29-p4d-debug-worker-0:2470955:2474561 [0] NCCL INFO [Proxy Service] Device 7 CPU core 77 +ywang29-p4d-debug-worker-0:2470955:2474562 [0] NCCL INFO [Proxy Service UDS] Device 7 CPU core 79 +ywang29-p4d-debug-worker-0:2470953:2470953 [5] NCCL INFO Check P2P Type isAllDirectP2p 1 directMode 0 isAllCudaP2p 1 +ywang29-p4d-debug-worker-0:2470953:2474563 [0] NCCL INFO [Proxy Service] Device 5 CPU core 32 +ywang29-p4d-debug-worker-0:2470953:2474564 [0] NCCL INFO [Proxy Service UDS] Device 5 CPU core 46 +ywang29-p4d-debug-worker-0:2470954:2470954 [6] NCCL INFO Check P2P Type isAllDirectP2p 1 directMode 0 isAllCudaP2p 1 +ywang29-p4d-debug-worker-0:2470954:2474565 [0] NCCL INFO [Proxy Service] Device 6 CPU core 93 +ywang29-p4d-debug-worker-0:2470954:2474566 [0] NCCL INFO [Proxy Service UDS] Device 6 CPU core 88 +ywang29-p4d-debug-worker-0:2470949:2470949 [2] NCCL INFO Check P2P Type isAllDirectP2p 1 directMode 0 isAllCudaP2p 1 +ywang29-p4d-debug-worker-0:2470949:2474567 [0] NCCL INFO [Proxy Service] Device 2 CPU core 1 +ywang29-p4d-debug-worker-0:2470949:2474568 [0] NCCL INFO [Proxy Service UDS] Device 2 CPU core 53 +ywang29-p4d-debug-worker-0:2470947:2470947 [0] NCCL INFO NET/OFI NCCL_OFI_TUNER is not available for platform : p4de.24xlarge, Fall back to NCCL's tuner +ywang29-p4d-debug-worker-0:2470947:2470947 [0] NCCL INFO threadThresholds 8/8/64 | 64/8/64 | 512 | 512 +ywang29-p4d-debug-worker-0:2470947:2470947 [0] NCCL INFO 24 coll channels, 24 collnet channels, 0 nvls channels, 32 p2p channels, 32 p2p channels per peer +ywang29-p4d-debug-worker-0:2470947:2470947 [0] NCCL INFO CC Off, workFifoBytes 1048576 +ywang29-p4d-debug-worker-0:2470949:2470949 [2] NCCL INFO NET/OFI NCCL_OFI_TUNER is not available for platform : p4de.24xlarge, Fall back to NCCL's tuner +ywang29-p4d-debug-worker-0:2470949:2470949 [2] NCCL INFO threadThresholds 8/8/64 | 64/8/64 | 512 | 512 +ywang29-p4d-debug-worker-0:2470949:2470949 [2] NCCL INFO 24 coll channels, 24 collnet channels, 0 nvls channels, 32 p2p channels, 32 p2p channels per peer +ywang29-p4d-debug-worker-0:2470953:2470953 [5] NCCL INFO NET/OFI NCCL_OFI_TUNER is not available for platform : p4de.24xlarge, Fall back to NCCL's tuner +ywang29-p4d-debug-worker-0:2470953:2470953 [5] NCCL INFO threadThresholds 8/8/64 | 64/8/64 | 512 | 512 +ywang29-p4d-debug-worker-0:2470953:2470953 [5] NCCL INFO 24 coll channels, 24 collnet channels, 0 nvls channels, 32 p2p channels, 32 p2p channels per peer +ywang29-p4d-debug-worker-0:2470948:2470948 [1] NCCL INFO NET/OFI NCCL_OFI_TUNER is not available for platform : p4de.24xlarge, Fall back to NCCL's tuner +ywang29-p4d-debug-worker-0:2470948:2470948 [1] NCCL INFO threadThresholds 8/8/64 | 64/8/64 | 512 | 512 +ywang29-p4d-debug-worker-0:2470948:2470948 [1] NCCL INFO 24 coll channels, 24 collnet channels, 0 nvls channels, 32 p2p channels, 32 p2p channels per peer +ywang29-p4d-debug-worker-0:2470954:2470954 [6] NCCL INFO NET/OFI NCCL_OFI_TUNER is not available for platform : p4de.24xlarge, Fall back to NCCL's tuner +ywang29-p4d-debug-worker-0:2470954:2470954 [6] NCCL INFO threadThresholds 8/8/64 | 64/8/64 | 512 | 512 +ywang29-p4d-debug-worker-0:2470954:2470954 [6] NCCL INFO 24 coll channels, 24 collnet channels, 0 nvls channels, 32 p2p channels, 32 p2p channels per peer +ywang29-p4d-debug-worker-0:2470951:2470951 [3] NCCL INFO NET/OFI NCCL_OFI_TUNER is not available for platform : p4de.24xlarge, Fall back to NCCL's tuner +ywang29-p4d-debug-worker-0:2470951:2470951 [3] NCCL INFO threadThresholds 8/8/64 | 64/8/64 | 512 | 512 +ywang29-p4d-debug-worker-0:2470951:2470951 [3] NCCL INFO 24 coll channels, 24 collnet channels, 0 nvls channels, 32 p2p channels, 32 p2p channels per peer +ywang29-p4d-debug-worker-0:2470952:2470952 [4] NCCL INFO NET/OFI NCCL_OFI_TUNER is not available for platform : p4de.24xlarge, Fall back to NCCL's tuner +ywang29-p4d-debug-worker-0:2470952:2470952 [4] NCCL INFO threadThresholds 8/8/64 | 64/8/64 | 512 | 512 +ywang29-p4d-debug-worker-0:2470952:2470952 [4] NCCL INFO 24 coll channels, 24 collnet channels, 0 nvls channels, 32 p2p channels, 32 p2p channels per peer +ywang29-p4d-debug-worker-0:2470955:2470955 [7] NCCL INFO NET/OFI NCCL_OFI_TUNER is not available for platform : p4de.24xlarge, Fall back to NCCL's tuner +ywang29-p4d-debug-worker-0:2470955:2470955 [7] NCCL INFO threadThresholds 8/8/64 | 64/8/64 | 512 | 512 +ywang29-p4d-debug-worker-0:2470955:2470955 [7] NCCL INFO 24 coll channels, 24 collnet channels, 0 nvls channels, 32 p2p channels, 32 p2p channels per peer +ywang29-p4d-debug-worker-0:2470952:2470952 [4] NCCL INFO ncclCommSplit comm 0x5647135d02c0 rank 4 nranks 8 cudaDev 4 nvmlDev 4 busId 901c0 parent 0x5647078795e0 splitCount 1 color 1266629538 key 4 - Init COMPLETE +ywang29-p4d-debug-worker-0:2470949:2470949 [2] NCCL INFO ncclCommSplit comm 0x56028de3b5a0 rank 2 nranks 8 cudaDev 2 nvmlDev 2 busId 201c0 parent 0x5602809d0750 splitCount 1 color 1266629538 key 2 - Init COMPLETE +ywang29-p4d-debug-worker-0:2470947:2470947 [0] NCCL INFO ncclCommSplit comm 0x55ffc26dd150 rank 0 nranks 8 cudaDev 0 nvmlDev 0 busId 101c0 parent 0x55ffbe0f7c80 splitCount 1 color 1266629538 key 0 - Init COMPLETE +ywang29-p4d-debug-worker-0:2470952:2470952 [4] NCCL INFO Init timings - ncclCommSplit: rank 4 nranks 8 total 0.18 (kernels 0.00, alloc 0.00, bootstrap 0.00, allgathers 0.01, topo 0.04, graphs 0.00, connections 0.07, rest 0.06) +ywang29-p4d-debug-worker-0:2470947:2470947 [0] NCCL INFO Init timings - ncclCommSplit: rank 0 nranks 8 total 0.24 (kernels 0.00, alloc 0.00, bootstrap 0.00, allgathers 0.00, topo 0.05, graphs 0.00, connections 0.06, rest 0.13) +ywang29-p4d-debug-worker-0:2470949:2470949 [2] NCCL INFO Init timings - ncclCommSplit: rank 2 nranks 8 total 0.15 (kernels 0.00, alloc 0.00, bootstrap 0.00, allgathers 0.01, topo 0.04, graphs 0.00, connections 0.06, rest 0.03) +ywang29-p4d-debug-worker-0:2470954:2470954 [6] NCCL INFO ncclCommSplit comm 0x560722415b60 rank 6 nranks 8 cudaDev 6 nvmlDev 6 busId a01c0 parent 0x56071a5c6320 splitCount 1 color 1266629538 key 6 - Init COMPLETE +ywang29-p4d-debug-worker-0:2470954:2470954 [6] NCCL INFO Init timings - ncclCommSplit: rank 6 nranks 8 total 0.25 (kernels 0.00, alloc 0.00, bootstrap 0.00, allgathers 0.02, topo 0.04, graphs 0.00, connections 0.07, rest 0.13) +ywang29-p4d-debug-worker-0:2470953:2470953 [5] NCCL INFO ncclCommSplit comm 0x55c02c235cf0 rank 5 nranks 8 cudaDev 5 nvmlDev 5 busId 901d0 parent 0x55c0201d0a60 splitCount 1 color 1266629538 key 5 - Init COMPLETE +ywang29-p4d-debug-worker-0:2470948:2470948 [1] NCCL INFO ncclCommSplit comm 0x558d03437a10 rank 1 nranks 8 cudaDev 1 nvmlDev 1 busId 101d0 parent 0x558cf7486a10 splitCount 1 color 1266629538 key 1 - Init COMPLETE +ywang29-p4d-debug-worker-0:2470951:2470951 [3] NCCL INFO ncclCommSplit comm 0x5612df7ca6e0 rank 3 nranks 8 cudaDev 3 nvmlDev 3 busId 201d0 parent 0x5612d38dcbf0 splitCount 1 color 1266629538 key 3 - Init COMPLETE +ywang29-p4d-debug-worker-0:2470953:2470953 [5] NCCL INFO Init timings - ncclCommSplit: rank 5 nranks 8 total 0.28 (kernels 0.00, alloc 0.00, bootstrap 0.00, allgathers 0.01, topo 0.04, graphs 0.00, connections 0.07, rest 0.16) +ywang29-p4d-debug-worker-0:2470948:2470948 [1] NCCL INFO Init timings - ncclCommSplit: rank 1 nranks 8 total 0.26 (kernels 0.00, alloc 0.00, bootstrap 0.00, allgathers 0.01, topo 0.04, graphs 0.00, connections 0.07, rest 0.14) +ywang29-p4d-debug-worker-0:2470955:2470955 [7] NCCL INFO ncclCommSplit comm 0x55e3778f3800 rank 7 nranks 8 cudaDev 7 nvmlDev 7 busId a01d0 parent 0x55e36d363080 splitCount 1 color 1266629538 key 7 - Init COMPLETE +ywang29-p4d-debug-worker-0:2470951:2470951 [3] NCCL INFO Init timings - ncclCommSplit: rank 3 nranks 8 total 0.16 (kernels 0.00, alloc 0.00, bootstrap 0.00, allgathers 0.01, topo 0.04, graphs 0.00, connections 0.07, rest 0.03) +ywang29-p4d-debug-worker-0:2470955:2470955 [7] NCCL INFO Init timings - ncclCommSplit: rank 7 nranks 8 total 0.18 (kernels 0.00, alloc 0.00, bootstrap 0.00, allgathers 0.01, topo 0.04, graphs 0.00, connections 0.07, rest 0.05) +ywang29-p4d-debug-worker-0:2470953:2470953 [5] NCCL INFO Channel 00/0 : 5[5] -> 6[6] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470953:2470953 [5] NCCL INFO Channel 01/0 : 5[5] -> 6[6] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470948:2470948 [1] NCCL INFO Channel 00/0 : 1[1] -> 2[2] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470953:2470953 [5] NCCL INFO Channel 02/0 : 5[5] -> 6[6] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470949:2470949 [2] NCCL INFO Channel 00/0 : 2[2] -> 3[3] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470954:2470954 [6] NCCL INFO Channel 00/0 : 6[6] -> 7[7] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470953:2470953 [5] NCCL INFO Channel 03/0 : 5[5] -> 6[6] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470948:2470948 [1] NCCL INFO Channel 01/0 : 1[1] -> 2[2] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470954:2470954 [6] NCCL INFO Channel 01/0 : 6[6] -> 7[7] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470949:2470949 [2] NCCL INFO Channel 01/0 : 2[2] -> 3[3] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470954:2470954 [6] NCCL INFO Channel 02/0 : 6[6] -> 7[7] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470949:2470949 [2] NCCL INFO Channel 02/0 : 2[2] -> 3[3] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470951:2470951 [3] NCCL INFO Channel 00/0 : 3[3] -> 4[4] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470949:2470949 [2] NCCL INFO Channel 03/0 : 2[2] -> 3[3] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470954:2470954 [6] NCCL INFO Channel 03/0 : 6[6] -> 7[7] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470948:2470948 [1] NCCL INFO Channel 02/0 : 1[1] -> 2[2] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470948:2470948 [1] NCCL INFO Channel 03/0 : 1[1] -> 2[2] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470952:2470952 [4] NCCL INFO Channel 00/0 : 4[4] -> 5[5] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470954:2470954 [6] NCCL INFO Channel 04/0 : 6[6] -> 7[7] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470953:2470953 [5] NCCL INFO Channel 04/0 : 5[5] -> 6[6] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470953:2470953 [5] NCCL INFO Channel 05/0 : 5[5] -> 6[6] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470951:2470951 [3] NCCL INFO Channel 01/0 : 3[3] -> 4[4] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470955:2470955 [7] NCCL INFO Channel 00/0 : 7[7] -> 0[0] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470948:2470948 [1] NCCL INFO Channel 04/0 : 1[1] -> 2[2] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470954:2470954 [6] NCCL INFO Channel 05/0 : 6[6] -> 7[7] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470952:2470952 [4] NCCL INFO Channel 01/0 : 4[4] -> 5[5] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470954:2470954 [6] NCCL INFO Channel 06/0 : 6[6] -> 7[7] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470949:2470949 [2] NCCL INFO Channel 04/0 : 2[2] -> 3[3] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470955:2470955 [7] NCCL INFO Channel 01/0 : 7[7] -> 0[0] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470948:2470948 [1] NCCL INFO Channel 05/0 : 1[1] -> 2[2] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470952:2470952 [4] NCCL INFO Channel 02/0 : 4[4] -> 5[5] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470947:2470947 [0] NCCL INFO Channel 00/0 : 0[0] -> 1[1] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470954:2470954 [6] NCCL INFO Channel 07/0 : 6[6] -> 7[7] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470953:2470953 [5] NCCL INFO Channel 06/0 : 5[5] -> 6[6] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470952:2470952 [4] NCCL INFO Channel 03/0 : 4[4] -> 5[5] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470948:2470948 [1] NCCL INFO Channel 06/0 : 1[1] -> 2[2] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470955:2470955 [7] NCCL INFO Channel 02/0 : 7[7] -> 0[0] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470947:2470947 [0] NCCL INFO Channel 01/0 : 0[0] -> 1[1] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470951:2470951 [3] NCCL INFO Channel 02/0 : 3[3] -> 4[4] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470951:2470951 [3] NCCL INFO Channel 03/0 : 3[3] -> 4[4] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470953:2470953 [5] NCCL INFO Channel 07/0 : 5[5] -> 6[6] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470955:2470955 [7] NCCL INFO Channel 03/0 : 7[7] -> 0[0] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470948:2470948 [1] NCCL INFO Channel 07/0 : 1[1] -> 2[2] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470947:2470947 [0] NCCL INFO Channel 02/0 : 0[0] -> 1[1] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470949:2470949 [2] NCCL INFO Channel 05/0 : 2[2] -> 3[3] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470952:2470952 [4] NCCL INFO Channel 04/0 : 4[4] -> 5[5] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470948:2470948 [1] NCCL INFO Channel 08/0 : 1[1] -> 2[2] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470954:2470954 [6] NCCL INFO Channel 08/0 : 6[6] -> 7[7] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470947:2470947 [0] NCCL INFO Channel 03/0 : 0[0] -> 1[1] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470953:2470953 [5] NCCL INFO Channel 08/0 : 5[5] -> 6[6] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470949:2470949 [2] NCCL INFO Channel 06/0 : 2[2] -> 3[3] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470955:2470955 [7] NCCL INFO Channel 04/0 : 7[7] -> 0[0] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470951:2470951 [3] NCCL INFO Channel 04/0 : 3[3] -> 4[4] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470949:2470949 [2] NCCL INFO Channel 07/0 : 2[2] -> 3[3] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470953:2470953 [5] NCCL INFO Channel 09/0 : 5[5] -> 6[6] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470948:2470948 [1] NCCL INFO Channel 09/0 : 1[1] -> 2[2] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470955:2470955 [7] NCCL INFO Channel 05/0 : 7[7] -> 0[0] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470947:2470947 [0] NCCL INFO Channel 04/0 : 0[0] -> 1[1] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470952:2470952 [4] NCCL INFO Channel 05/0 : 4[4] -> 5[5] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470954:2470954 [6] NCCL INFO Channel 09/0 : 6[6] -> 7[7] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470954:2470954 [6] NCCL INFO Channel 10/0 : 6[6] -> 7[7] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470951:2470951 [3] NCCL INFO Channel 05/0 : 3[3] -> 4[4] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470952:2470952 [4] NCCL INFO Channel 06/0 : 4[4] -> 5[5] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470955:2470955 [7] NCCL INFO Channel 06/0 : 7[7] -> 0[0] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470947:2470947 [0] NCCL INFO Channel 05/0 : 0[0] -> 1[1] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470953:2470953 [5] NCCL INFO Channel 10/0 : 5[5] -> 6[6] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470948:2470948 [1] NCCL INFO Channel 10/0 : 1[1] -> 2[2] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470949:2470949 [2] NCCL INFO Channel 08/0 : 2[2] -> 3[3] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470955:2470955 [7] NCCL INFO Channel 07/0 : 7[7] -> 0[0] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470947:2470947 [0] NCCL INFO Channel 06/0 : 0[0] -> 1[1] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470953:2470953 [5] NCCL INFO Channel 11/0 : 5[5] -> 6[6] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470952:2470952 [4] NCCL INFO Channel 07/0 : 4[4] -> 5[5] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470954:2470954 [6] NCCL INFO Channel 11/0 : 6[6] -> 7[7] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470948:2470948 [1] NCCL INFO Channel 11/0 : 1[1] -> 2[2] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470953:2470953 [5] NCCL INFO Channel 12/0 : 5[5] -> 6[6] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470951:2470951 [3] NCCL INFO Channel 06/0 : 3[3] -> 4[4] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470952:2470952 [4] NCCL INFO Channel 08/0 : 4[4] -> 5[5] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470947:2470947 [0] NCCL INFO Channel 07/0 : 0[0] -> 1[1] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470955:2470955 [7] NCCL INFO Channel 08/0 : 7[7] -> 0[0] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470953:2470953 [5] NCCL INFO Channel 13/0 : 5[5] -> 6[6] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470947:2470947 [0] NCCL INFO Channel 08/0 : 0[0] -> 1[1] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470955:2470955 [7] NCCL INFO Channel 09/0 : 7[7] -> 0[0] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470949:2470949 [2] NCCL INFO Channel 09/0 : 2[2] -> 3[3] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470952:2470952 [4] NCCL INFO Channel 09/0 : 4[4] -> 5[5] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470951:2470951 [3] NCCL INFO Channel 07/0 : 3[3] -> 4[4] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470954:2470954 [6] NCCL INFO Channel 12/0 : 6[6] -> 7[7] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470954:2470954 [6] NCCL INFO Channel 13/0 : 6[6] -> 7[7] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470948:2470948 [1] NCCL INFO Channel 12/0 : 1[1] -> 2[2] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470952:2470952 [4] NCCL INFO Channel 10/0 : 4[4] -> 5[5] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470951:2470951 [3] NCCL INFO Channel 08/0 : 3[3] -> 4[4] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470953:2470953 [5] NCCL INFO Channel 14/0 : 5[5] -> 6[6] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470949:2470949 [2] NCCL INFO Channel 10/0 : 2[2] -> 3[3] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470947:2470947 [0] NCCL INFO Channel 09/0 : 0[0] -> 1[1] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470954:2470954 [6] NCCL INFO Channel 14/0 : 6[6] -> 7[7] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470947:2470947 [0] NCCL INFO Channel 10/0 : 0[0] -> 1[1] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470955:2470955 [7] NCCL INFO Channel 10/0 : 7[7] -> 0[0] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470949:2470949 [2] NCCL INFO Channel 11/0 : 2[2] -> 3[3] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470953:2470953 [5] NCCL INFO Channel 15/0 : 5[5] -> 6[6] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470951:2470951 [3] NCCL INFO Channel 09/0 : 3[3] -> 4[4] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470948:2470948 [1] NCCL INFO Channel 13/0 : 1[1] -> 2[2] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470954:2470954 [6] NCCL INFO Channel 15/0 : 6[6] -> 7[7] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470951:2470951 [3] NCCL INFO Channel 10/0 : 3[3] -> 4[4] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470952:2470952 [4] NCCL INFO Channel 11/0 : 4[4] -> 5[5] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470955:2470955 [7] NCCL INFO Channel 11/0 : 7[7] -> 0[0] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470949:2470949 [2] NCCL INFO Channel 12/0 : 2[2] -> 3[3] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470953:2470953 [5] NCCL INFO Channel 16/0 : 5[5] -> 6[6] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470948:2470948 [1] NCCL INFO Channel 14/0 : 1[1] -> 2[2] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470947:2470947 [0] NCCL INFO Channel 11/0 : 0[0] -> 1[1] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470955:2470955 [7] NCCL INFO Channel 12/0 : 7[7] -> 0[0] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470949:2470949 [2] NCCL INFO Channel 13/0 : 2[2] -> 3[3] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470954:2470954 [6] NCCL INFO Channel 16/0 : 6[6] -> 7[7] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470953:2470953 [5] NCCL INFO Channel 17/0 : 5[5] -> 6[6] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470952:2470952 [4] NCCL INFO Channel 12/0 : 4[4] -> 5[5] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470953:2470953 [5] NCCL INFO Channel 18/0 : 5[5] -> 6[6] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470951:2470951 [3] NCCL INFO Channel 11/0 : 3[3] -> 4[4] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470954:2470954 [6] NCCL INFO Channel 17/0 : 6[6] -> 7[7] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470949:2470949 [2] NCCL INFO Channel 14/0 : 2[2] -> 3[3] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470955:2470955 [7] NCCL INFO Channel 13/0 : 7[7] -> 0[0] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470952:2470952 [4] NCCL INFO Channel 13/0 : 4[4] -> 5[5] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470952:2470952 [4] NCCL INFO Channel 14/0 : 4[4] -> 5[5] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470948:2470948 [1] NCCL INFO Channel 15/0 : 1[1] -> 2[2] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470954:2470954 [6] NCCL INFO Channel 18/0 : 6[6] -> 7[7] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470955:2470955 [7] NCCL INFO Channel 14/0 : 7[7] -> 0[0] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470947:2470947 [0] NCCL INFO Channel 12/0 : 0[0] -> 1[1] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470953:2470953 [5] NCCL INFO Channel 19/0 : 5[5] -> 6[6] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470949:2470949 [2] NCCL INFO Channel 15/0 : 2[2] -> 3[3] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470953:2470953 [5] NCCL INFO Channel 20/0 : 5[5] -> 6[6] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470951:2470951 [3] NCCL INFO Channel 12/0 : 3[3] -> 4[4] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470955:2470955 [7] NCCL INFO Channel 15/0 : 7[7] -> 0[0] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470949:2470949 [2] NCCL INFO Channel 16/0 : 2[2] -> 3[3] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470954:2470954 [6] NCCL INFO Channel 19/0 : 6[6] -> 7[7] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470947:2470947 [0] NCCL INFO Channel 13/0 : 0[0] -> 1[1] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470952:2470952 [4] NCCL INFO Channel 15/0 : 4[4] -> 5[5] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470951:2470951 [3] NCCL INFO Channel 13/0 : 3[3] -> 4[4] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470948:2470948 [1] NCCL INFO Channel 16/0 : 1[1] -> 2[2] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470954:2470954 [6] NCCL INFO Channel 20/0 : 6[6] -> 7[7] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470947:2470947 [0] NCCL INFO Channel 14/0 : 0[0] -> 1[1] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470952:2470952 [4] NCCL INFO Channel 16/0 : 4[4] -> 5[5] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470949:2470949 [2] NCCL INFO Channel 17/0 : 2[2] -> 3[3] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470953:2470953 [5] NCCL INFO Channel 21/0 : 5[5] -> 6[6] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470955:2470955 [7] NCCL INFO Channel 16/0 : 7[7] -> 0[0] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470947:2470947 [0] NCCL INFO Channel 15/0 : 0[0] -> 1[1] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470952:2470952 [4] NCCL INFO Channel 17/0 : 4[4] -> 5[5] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470949:2470949 [2] NCCL INFO Channel 18/0 : 2[2] -> 3[3] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470954:2470954 [6] NCCL INFO Channel 21/0 : 6[6] -> 7[7] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470953:2470953 [5] NCCL INFO Channel 22/0 : 5[5] -> 6[6] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470954:2470954 [6] NCCL INFO Channel 22/0 : 6[6] -> 7[7] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470952:2470952 [4] NCCL INFO Channel 18/0 : 4[4] -> 5[5] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470948:2470948 [1] NCCL INFO Channel 17/0 : 1[1] -> 2[2] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470953:2470953 [5] NCCL INFO Channel 23/0 : 5[5] -> 6[6] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470951:2470951 [3] NCCL INFO Channel 14/0 : 3[3] -> 4[4] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470949:2470949 [2] NCCL INFO Channel 19/0 : 2[2] -> 3[3] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470954:2470954 [6] NCCL INFO Channel 23/0 : 6[6] -> 7[7] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470949:2470949 [2] NCCL INFO Channel 20/0 : 2[2] -> 3[3] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470955:2470955 [7] NCCL INFO Channel 17/0 : 7[7] -> 0[0] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470948:2470948 [1] NCCL INFO Channel 18/0 : 1[1] -> 2[2] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470951:2470951 [3] NCCL INFO Channel 15/0 : 3[3] -> 4[4] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470951:2470951 [3] NCCL INFO Channel 16/0 : 3[3] -> 4[4] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470947:2470947 [0] NCCL INFO Channel 16/0 : 0[0] -> 1[1] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470952:2470952 [4] NCCL INFO Channel 19/0 : 4[4] -> 5[5] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470948:2470948 [1] NCCL INFO Channel 19/0 : 1[1] -> 2[2] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470955:2470955 [7] NCCL INFO Channel 18/0 : 7[7] -> 0[0] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470949:2470949 [2] NCCL INFO Channel 21/0 : 2[2] -> 3[3] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470955:2470955 [7] NCCL INFO Channel 19/0 : 7[7] -> 0[0] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470948:2470948 [1] NCCL INFO Channel 20/0 : 1[1] -> 2[2] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470952:2470952 [4] NCCL INFO Channel 20/0 : 4[4] -> 5[5] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470952:2470952 [4] NCCL INFO Channel 21/0 : 4[4] -> 5[5] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470951:2470951 [3] NCCL INFO Channel 17/0 : 3[3] -> 4[4] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470955:2470955 [7] NCCL INFO Channel 20/0 : 7[7] -> 0[0] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470949:2470949 [2] NCCL INFO Channel 22/0 : 2[2] -> 3[3] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470948:2470948 [1] NCCL INFO Channel 21/0 : 1[1] -> 2[2] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470947:2470947 [0] NCCL INFO Channel 17/0 : 0[0] -> 1[1] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470951:2470951 [3] NCCL INFO Channel 18/0 : 3[3] -> 4[4] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470952:2470952 [4] NCCL INFO Channel 22/0 : 4[4] -> 5[5] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470947:2470947 [0] NCCL INFO Channel 18/0 : 0[0] -> 1[1] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470955:2470955 [7] NCCL INFO Channel 21/0 : 7[7] -> 0[0] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470949:2470949 [2] NCCL INFO Channel 23/0 : 2[2] -> 3[3] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470955:2470955 [7] NCCL INFO Channel 22/0 : 7[7] -> 0[0] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470951:2470951 [3] NCCL INFO Channel 19/0 : 3[3] -> 4[4] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470952:2470952 [4] NCCL INFO Channel 23/0 : 4[4] -> 5[5] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470947:2470947 [0] NCCL INFO Channel 19/0 : 0[0] -> 1[1] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470948:2470948 [1] NCCL INFO Channel 22/0 : 1[1] -> 2[2] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470948:2470948 [1] NCCL INFO Channel 23/0 : 1[1] -> 2[2] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470951:2470951 [3] NCCL INFO Channel 20/0 : 3[3] -> 4[4] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470947:2470947 [0] NCCL INFO Channel 20/0 : 0[0] -> 1[1] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470955:2470955 [7] NCCL INFO Channel 23/0 : 7[7] -> 0[0] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470951:2470951 [3] NCCL INFO Channel 21/0 : 3[3] -> 4[4] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470947:2470947 [0] NCCL INFO Channel 21/0 : 0[0] -> 1[1] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470951:2470951 [3] NCCL INFO Channel 22/0 : 3[3] -> 4[4] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470947:2470947 [0] NCCL INFO Channel 22/0 : 0[0] -> 1[1] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470951:2470951 [3] NCCL INFO Channel 23/0 : 3[3] -> 4[4] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470947:2470947 [0] NCCL INFO Channel 23/0 : 0[0] -> 1[1] via P2P/CUMEM/read +ywang29-p4d-debug-worker-0:2470949:2470949 [2] NCCL INFO Connected all rings, use ring PXN 0 GDR 1 +ywang29-p4d-debug-worker-0:2470951:2470951 [3] NCCL INFO Connected all rings, use ring PXN 0 GDR 1 +ywang29-p4d-debug-worker-0:2470953:2470953 [5] NCCL INFO Connected all rings, use ring PXN 0 GDR 1 +ywang29-p4d-debug-worker-0:2470954:2470954 [6] NCCL INFO Connected all rings, use ring PXN 0 GDR 1 +ywang29-p4d-debug-worker-0:2470948:2470948 [1] NCCL INFO Connected all rings, use ring PXN 0 GDR 1 +ywang29-p4d-debug-worker-0:2470952:2470952 [4] NCCL INFO Connected all rings, use ring PXN 0 GDR 1 +ywang29-p4d-debug-worker-0:2470955:2470955 [7] NCCL INFO Connected all rings, use ring PXN 0 GDR 1 +ywang29-p4d-debug-worker-0:2470947:2470947 [0] NCCL INFO Connected all rings, use ring PXN 0 GDR 1 +Before initializing optimizer states +MA 4.73 GB Max_MA 4.73 GB CA 4.77 GB Max_CA 5 GB +CPU Virtual Memory: used = 70.92 GB, percent = 6.3% +After initializing optimizer states +MA 4.73 GB Max_MA 5.68 GB CA 5.71 GB Max_CA 6 GB +CPU Virtual Memory: used = 70.92 GB, percent = 6.3% +After initializing ZeRO optimizer +MA 4.73 GB Max_MA 4.73 GB CA 5.71 GB Max_CA 6 GB +CPU Virtual Memory: used = 70.92 GB, percent = 6.3% +[INFO|trainer.py:1587] 2026-04-29 15:44:34,254 >> ***** Running training ***** +[INFO|trainer.py:1588] 2026-04-29 15:44:34,254 >> Num examples = 60,000 +[INFO|trainer.py:1589] 2026-04-29 15:44:34,254 >> Num Epochs = 2 +[INFO|trainer.py:1590] 2026-04-29 15:44:34,254 >> Instantaneous batch size per device = 2 +[INFO|trainer.py:1593] 2026-04-29 15:44:34,254 >> Total train batch size (w. parallel, distributed & accumulation) = 128 +[INFO|trainer.py:1594] 2026-04-29 15:44:34,254 >> Gradient Accumulation steps = 8 +[INFO|trainer.py:1595] 2026-04-29 15:44:34,254 >> Total optimization steps = 938 +[INFO|trainer.py:1596] 2026-04-29 15:44:34,255 >> Number of trainable parameters = 2,031,739,904 +wandb: [wandb.login()] Loaded credentials for https://api.wandb.ai from WANDB_API_KEY. +wandb: Currently logged in as: kkhya (maskmoe) to https://api.wandb.ai. Use `wandb login --relogin` to force relogin +wandb: setting up run jl6u15v5 +wandb: Tracking run with wandb version 0.25.1 +wandb: Run data is saved locally in /nfs/ywang29/lm-factory/wandb/run-20260429_154434-jl6u15v5 +wandb: Run `wandb offline` to turn off syncing. +wandb: Syncing run qwen3_1.7b_math_fft +wandb: ⭐️ View project at https://wandb.ai/maskmoe/MFT-LM +wandb: 🚀 View run at https://wandb.ai/maskmoe/MFT-LM/runs/jl6u15v5 + 0%| | 0/938 [00:00> Saving model checkpoint to saves/qwen3_1.7b/math/fft/checkpoint-500 +[INFO|configuration_utils.py:432] 2026-04-29 16:19:18,055 >> Configuration saved in saves/qwen3_1.7b/math/fft/checkpoint-500/config.json +[INFO|configuration_utils.py:803] 2026-04-29 16:19:18,058 >> Configuration saved in saves/qwen3_1.7b/math/fft/checkpoint-500/generation_config.json + + Writing model shards: 0%| | 0/1 [00:00> Model weights saved in saves/qwen3_1.7b/math/fft/checkpoint-500/model.safetensors +[INFO|tokenization_utils_base.py:3224] 2026-04-29 16:19:24,380 >> chat template saved in saves/qwen3_1.7b/math/fft/checkpoint-500/chat_template.jinja +[INFO|tokenization_utils_base.py:2078] 2026-04-29 16:19:24,384 >> tokenizer config file saved in saves/qwen3_1.7b/math/fft/checkpoint-500/tokenizer_config.json + 53%|█████▎ | 501/938 [34:52<51:35, 7.08s/it] 54%|█████▎ | 502/938 [34:56<44:10, 6.08s/it] 54%|█████▎ | 503/938 [34:59<38:54, 5.37s/it] 54%|█████▎ | 504/938 [35:04<36:57, 5.11s/it] 54%|█████▍ | 505/938 [35:08<35:11, 4.88s/it] 54%|█████▍ | 506/938 [35:13<33:51, 4.70s/it] 54%|█████▍ | 507/938 [35:17<32:40, 4.55s/it] 54%|█████▍ | 508/938 [35:22<33:33, 4.68s/it] 54%|█████▍ | 509/938 [35:26<32:07, 4.49s/it] 54%|█████▍ | 510/938 [35:30<31:53, 4.47s/it] {'loss': '0.2784', 'grad_norm': '0.4592', 'learning_rate': '5.13e-06', 'epoch': '1.087'} + 54%|█████▍ | 510/938 [35:30<31:53, 4.47s/it] 54%|█████▍ | 511/938 [35:34<30:50, 4.33s/it] 55%|█████▍ | 512/938 [35:38<30:31, 4.30s/it] 55%|█████▍ | 513/938 [35:44<32:17, 4.56s/it] 55%|█████▍ | 514/938 [35:48<31:03, 4.39s/it] 55%|█████▍ | 515/938 [35:51<29:44, 4.22s/it] 55%|█████▌ | 516/938 [35:56<30:52, 4.39s/it] 55%|█████▌ | 517/938 [36:00<29:49, 4.25s/it] 55%|█████▌ | 518/938 [36:04<29:27, 4.21s/it] 55%|█████▌ | 519/938 [36:09<31:01, 4.44s/it] 55%|█████▌ | 520/938 [36:13<29:14, 4.20s/it] {'loss': '0.2618', 'grad_norm': '0.4609', 'learning_rate': '4.944e-06', 'epoch': '1.109'} + 55%|█████▌ | 520/938 [36:13<29:14, 4.20s/it] 56%|█████▌ | 521/938 [36:17<28:34, 4.11s/it] 56%|█████▌ | 522/938 [36:21<29:05, 4.20s/it] 56%|█████▌ | 523/938 [36:25<29:14, 4.23s/it] 56%|█████▌ | 524/938 [36:30<29:51, 4.33s/it] 56%|█████▌ | 525/938 [36:34<29:14, 4.25s/it] 56%|█████▌ | 526/938 [36:38<27:33, 4.01s/it] 56%|█████▌ | 527/938 [36:41<26:31, 3.87s/it] 56%|█████▋ | 528/938 [36:45<27:25, 4.01s/it] 56%|█████▋ | 529/938 [36:50<28:01, 4.11s/it] 57%|█████▋ | 530/938 [36:54<27:16, 4.01s/it] {'loss': '0.2712', 'grad_norm': '0.4843', 'learning_rate': '4.758e-06', 'epoch': '1.13'} + 57%|█████▋ | 530/938 [36:54<27:16, 4.01s/it] 57%|█████▋ | 531/938 [36:58<27:51, 4.11s/it] 57%|█████▋ | 532/938 [37:03<30:17, 4.48s/it] 57%|█████▋ | 533/938 [37:07<28:37, 4.24s/it] 57%|█████▋ | 534/938 [37:11<27:44, 4.12s/it] 57%|█████▋ | 535/938 [37:15<27:43, 4.13s/it] 57%|█████▋ | 536/938 [37:20<29:20, 4.38s/it] 57%|█████▋ | 537/938 [37:25<30:00, 4.49s/it] 57%|█████▋ | 538/938 [37:29<29:09, 4.37s/it] 57%|█████▋ | 539/938 [37:33<29:10, 4.39s/it] 58%|█████▊ | 540/938 [37:37<27:22, 4.13s/it] {'loss': '0.2627', 'grad_norm': '0.5324', 'learning_rate': '4.572e-06', 'epoch': '1.151'} + 58%|█████▊ | 540/938 [37:37<27:22, 4.13s/it] 58%|█████▊ | 541/938 [37:41<28:24, 4.29s/it] 58%|█████▊ | 542/938 [37:45<27:53, 4.23s/it] 58%|█████▊ | 543/938 [37:49<26:48, 4.07s/it] 58%|█████▊ | 544/938 [37:53<26:41, 4.06s/it] 58%|█████▊ | 545/938 [37:58<27:16, 4.16s/it] 58%|█████▊ | 546/938 [38:01<26:23, 4.04s/it] 58%|█████▊ | 547/938 [38:05<26:15, 4.03s/it] 58%|█████▊ | 548/938 [38:10<27:35, 4.24s/it] 59%|█████▊ | 549/938 [38:14<26:45, 4.13s/it] 59%|█████▊ | 550/938 [38:18<26:52, 4.16s/it] {'loss': '0.2738', 'grad_norm': '0.4596', 'learning_rate': '4.387e-06', 'epoch': '1.173'} + 59%|█████▊ | 550/938 [38:18<26:52, 4.16s/it] 59%|█████▊ | 551/938 [38:22<26:25, 4.10s/it] 59%|█████▉ | 552/938 [38:26<26:24, 4.10s/it] 59%|█████▉ | 553/938 [38:30<26:34, 4.14s/it] 59%|█████▉ | 554/938 [38:34<25:32, 3.99s/it] 59%|█████▉ | 555/938 [38:38<25:14, 3.95s/it] 59%|█████▉ | 556/938 [38:42<25:23, 3.99s/it] 59%|█████▉ | 557/938 [38:46<24:56, 3.93s/it] 59%|█████▉ | 558/938 [38:50<24:26, 3.86s/it] 60%|█████▉ | 559/938 [38:54<25:26, 4.03s/it] 60%|█████▉ | 560/938 [38:58<24:56, 3.96s/it] {'loss': '0.2654', 'grad_norm': '0.4884', 'learning_rate': '4.203e-06', 'epoch': '1.194'} + 60%|█████▉ | 560/938 [38:58<24:56, 3.96s/it] 60%|█████▉ | 561/938 [39:01<24:21, 3.88s/it] 60%|█████▉ | 562/938 [39:06<25:09, 4.01s/it] 60%|██████ | 563/938 [39:10<24:51, 3.98s/it] 60%|██████ | 564/938 [39:13<24:14, 3.89s/it] 60%|██████ | 565/938 [39:18<24:45, 3.98s/it] 60%|██████ | 566/938 [39:21<24:34, 3.96s/it] 60%|██████ | 567/938 [39:26<24:52, 4.02s/it] 61%|██████ | 568/938 [39:30<25:31, 4.14s/it] 61%|██████ | 569/938 [39:35<27:18, 4.44s/it] 61%|██████ | 570/938 [39:40<28:20, 4.62s/it] {'loss': '0.2673', 'grad_norm': '0.4471', 'learning_rate': '4.02e-06', 'epoch': '1.215'} + 61%|██████ | 570/938 [39:40<28:20, 4.62s/it] 61%|██████ | 571/938 [39:44<26:47, 4.38s/it] 61%|██████ | 572/938 [39:48<25:11, 4.13s/it] 61%|██████ | 573/938 [39:52<25:18, 4.16s/it] 61%|██████ | 574/938 [39:55<23:45, 3.92s/it] 61%|██████▏ | 575/938 [40:00<24:30, 4.05s/it] 61%|██████▏ | 576/938 [40:03<24:09, 4.00s/it] 62%|██████▏ | 577/938 [40:08<24:54, 4.14s/it] 62%|██████▏ | 578/938 [40:12<24:08, 4.02s/it] 62%|██████▏ | 579/938 [40:16<23:52, 3.99s/it] 62%|██████▏ | 580/938 [40:20<24:08, 4.05s/it] {'loss': '0.263', 'grad_norm': '0.4469', 'learning_rate': '3.838e-06', 'epoch': '1.237'} + 62%|██████▏ | 580/938 [40:20<24:08, 4.05s/it] 62%|██████▏ | 581/938 [40:24<23:42, 3.99s/it] 62%|██████▏ | 582/938 [40:27<23:25, 3.95s/it] 62%|██████▏ | 583/938 [40:31<22:44, 3.85s/it] 62%|██████▏ | 584/938 [40:37<25:52, 4.38s/it] 62%|██████▏ | 585/938 [40:40<24:17, 4.13s/it] 62%|██████▏ | 586/938 [40:46<27:31, 4.69s/it] 63%|██████▎ | 587/938 [40:51<27:25, 4.69s/it] 63%|██████▎ | 588/938 [40:55<26:50, 4.60s/it] 63%|██████▎ | 589/938 [41:00<26:11, 4.50s/it] 63%|██████▎ | 590/938 [41:04<25:13, 4.35s/it] {'loss': '0.2718', 'grad_norm': '0.4668', 'learning_rate': '3.658e-06', 'epoch': '1.258'} + 63%|██████▎ | 590/938 [41:04<25:13, 4.35s/it] 63%|██████▎ | 591/938 [41:08<25:10, 4.35s/it] 63%|██████▎ | 592/938 [41:11<23:41, 4.11s/it] 63%|██████▎ | 593/938 [41:16<24:42, 4.30s/it] 63%|██████▎ | 594/938 [41:22<26:22, 4.60s/it] 63%|██████▎ | 595/938 [41:26<25:15, 4.42s/it] 64%|██████▎ | 596/938 [41:29<24:13, 4.25s/it] 64%|██████▎ | 597/938 [41:34<25:31, 4.49s/it] 64%|██████▍ | 598/938 [41:38<23:56, 4.22s/it] 64%|██████▍ | 599/938 [41:42<23:23, 4.14s/it] 64%|██████▍ | 600/938 [41:46<23:35, 4.19s/it] {'loss': '0.2709', 'grad_norm': '0.4404', 'learning_rate': '3.48e-06', 'epoch': '1.279'} + 64%|██████▍ | 600/938 [41:46<23:35, 4.19s/it] 64%|██████▍ | 601/938 [41:51<24:02, 4.28s/it] 64%|██████▍ | 602/938 [41:55<24:34, 4.39s/it] 64%|██████▍ | 603/938 [42:00<24:19, 4.36s/it] 64%|██████▍ | 604/938 [42:03<22:36, 4.06s/it] 64%|██████▍ | 605/938 [42:07<22:38, 4.08s/it] 65%|██████▍ | 606/938 [42:12<23:04, 4.17s/it] 65%|██████▍ | 607/938 [42:16<23:26, 4.25s/it] 65%|██████▍ | 608/938 [42:21<23:52, 4.34s/it] 65%|██████▍ | 609/938 [42:25<23:14, 4.24s/it] 65%|██████▌ | 610/938 [42:29<22:58, 4.20s/it] {'loss': '0.2683', 'grad_norm': '0.5088', 'learning_rate': '3.304e-06', 'epoch': '1.301'} + 65%|██████▌ | 610/938 [42:29<22:58, 4.20s/it] 65%|██████▌ | 611/938 [42:33<22:25, 4.11s/it] 65%|██████▌ | 612/938 [42:37<22:43, 4.18s/it] 65%|██████▌ | 613/938 [42:41<22:51, 4.22s/it] 65%|██████▌ | 614/938 [42:45<22:49, 4.23s/it] 66%|██████▌ | 615/938 [42:50<22:34, 4.19s/it] 66%|██████▌ | 616/938 [42:54<22:15, 4.15s/it] 66%|██████▌ | 617/938 [42:57<21:21, 3.99s/it] 66%|██████▌ | 618/938 [43:02<21:59, 4.12s/it] 66%|██████▌ | 619/938 [43:06<21:45, 4.09s/it] 66%|██████▌ | 620/938 [43:10<21:18, 4.02s/it] {'loss': '0.2715', 'grad_norm': '0.4963', 'learning_rate': '3.13e-06', 'epoch': '1.322'} + 66%|██████▌ | 620/938 [43:10<21:18, 4.02s/it] 66%|██████▌ | 621/938 [43:13<21:00, 3.98s/it] 66%|██████▋ | 622/938 [43:18<22:16, 4.23s/it] 66%|██████▋ | 623/938 [43:22<22:11, 4.23s/it] 67%|██████▋ | 624/938 [43:27<22:39, 4.33s/it] 67%|██████▋ | 625/938 [43:32<22:49, 4.37s/it] 67%|██████▋ | 626/938 [43:35<21:50, 4.20s/it] 67%|██████▋ | 627/938 [43:39<21:06, 4.07s/it] 67%|██████▋ | 628/938 [43:43<20:58, 4.06s/it] 67%|██████▋ | 629/938 [43:47<20:05, 3.90s/it] 67%|██████▋ | 630/938 [43:51<20:23, 3.97s/it] {'loss': '0.2726', 'grad_norm': '0.4921', 'learning_rate': '2.958e-06', 'epoch': '1.343'} + 67%|██████▋ | 630/938 [43:51<20:23, 3.97s/it] 67%|██████▋ | 631/938 [43:55<20:03, 3.92s/it] 67%|██████▋ | 632/938 [43:59<20:01, 3.93s/it] 67%|██████▋ | 633/938 [44:02<19:48, 3.90s/it] 68%|██████▊ | 634/938 [44:07<21:27, 4.24s/it] 68%|██████▊ | 635/938 [44:11<20:24, 4.04s/it] 68%|██████▊ | 636/938 [44:15<20:26, 4.06s/it] 68%|██████▊ | 637/938 [44:19<20:42, 4.13s/it] 68%|██████▊ | 638/938 [44:24<20:42, 4.14s/it] 68%|██████▊ | 639/938 [44:28<20:56, 4.20s/it] 68%|██████▊ | 640/938 [44:32<20:11, 4.07s/it] {'loss': '0.2665', 'grad_norm': '0.4858', 'learning_rate': '2.79e-06', 'epoch': '1.365'} + 68%|██████▊ | 640/938 [44:32<20:11, 4.07s/it] 68%|██████▊ | 641/938 [44:36<20:03, 4.05s/it] 68%|██████▊ | 642/938 [44:39<19:42, 3.99s/it] 69%|██████▊ | 643/938 [44:43<19:30, 3.97s/it] 69%|██████▊ | 644/938 [44:48<20:12, 4.12s/it] 69%|██████▉ | 645/938 [44:53<21:14, 4.35s/it] 69%|██████▉ | 646/938 [44:57<20:17, 4.17s/it] 69%|██████▉ | 647/938 [45:00<19:33, 4.03s/it] 69%|██████▉ | 648/938 [45:04<19:18, 4.00s/it] 69%|██████▉ | 649/938 [45:08<18:56, 3.93s/it] 69%|██████▉ | 650/938 [45:12<18:31, 3.86s/it] {'loss': '0.2647', 'grad_norm': '0.4581', 'learning_rate': '2.625e-06', 'epoch': '1.386'} + 69%|██████▉ | 650/938 [45:12<18:31, 3.86s/it] 69%|██████▉ | 651/938 [45:16<18:40, 3.90s/it] 70%|██████▉ | 652/938 [45:20<19:59, 4.19s/it] 70%|██████▉ | 653/938 [45:24<19:13, 4.05s/it] 70%|██████▉ | 654/938 [45:29<19:33, 4.13s/it] 70%|██████▉ | 655/938 [45:32<19:09, 4.06s/it] 70%|██████▉ | 656/938 [45:37<19:19, 4.11s/it] 70%|███████ | 657/938 [45:41<19:32, 4.17s/it] 70%|███████ | 658/938 [45:45<19:41, 4.22s/it] 70%|███████ | 659/938 [45:49<18:56, 4.07s/it] 70%|███████ | 660/938 [45:53<19:03, 4.11s/it] {'loss': '0.2691', 'grad_norm': '0.4541', 'learning_rate': '2.462e-06', 'epoch': '1.407'} + 70%|███████ | 660/938 [45:53<19:03, 4.11s/it] 70%|███████ | 661/938 [45:57<18:50, 4.08s/it] 71%|███████ | 662/938 [46:01<18:39, 4.06s/it] 71%|███████ | 663/938 [46:06<19:08, 4.18s/it] 71%|███████ | 664/938 [46:10<18:35, 4.07s/it] 71%|███████ | 665/938 [46:13<17:45, 3.90s/it] 71%|███████ | 666/938 [46:18<18:33, 4.09s/it] 71%|███████ | 667/938 [46:22<18:26, 4.08s/it] 71%|███████ | 668/938 [46:26<18:29, 4.11s/it] 71%|███████▏ | 669/938 [46:30<18:58, 4.23s/it] 71%|███████▏ | 670/938 [46:34<18:44, 4.20s/it] {'loss': '0.2636', 'grad_norm': '0.5011', 'learning_rate': '2.304e-06', 'epoch': '1.429'} + 71%|███████▏ | 670/938 [46:34<18:44, 4.20s/it] 72%|███████▏ | 671/938 [46:38<17:55, 4.03s/it] 72%|███████▏ | 672/938 [46:42<17:16, 3.90s/it] 72%|███████▏ | 673/938 [46:46<17:26, 3.95s/it] 72%|███████▏ | 674/938 [46:50<17:51, 4.06s/it] 72%|███████▏ | 675/938 [46:55<18:41, 4.27s/it] 72%|███████▏ | 676/938 [46:58<17:53, 4.10s/it] 72%|███████▏ | 677/938 [47:03<17:55, 4.12s/it] 72%|███████▏ | 678/938 [47:07<17:47, 4.11s/it] 72%|███████▏ | 679/938 [47:10<17:06, 3.96s/it] 72%|███████▏ | 680/938 [47:15<17:20, 4.03s/it] {'loss': '0.2607', 'grad_norm': '0.4419', 'learning_rate': '2.149e-06', 'epoch': '1.45'} + 72%|███████▏ | 680/938 [47:15<17:20, 4.03s/it] 73%|███████▎ | 681/938 [47:18<16:31, 3.86s/it] 73%|███████▎ | 682/938 [47:22<16:57, 3.97s/it] 73%|███████▎ | 683/938 [47:27<17:15, 4.06s/it] 73%|███████▎ | 684/938 [47:30<16:50, 3.98s/it] 73%|███████▎ | 685/938 [47:34<17:00, 4.03s/it] 73%|███████▎ | 686/938 [47:39<17:15, 4.11s/it] 73%|███████▎ | 687/938 [47:43<17:36, 4.21s/it] 73%|███████▎ | 688/938 [47:47<16:29, 3.96s/it] 73%|███████▎ | 689/938 [47:51<16:25, 3.96s/it] 74%|███████▎ | 690/938 [47:54<16:02, 3.88s/it] {'loss': '0.2723', 'grad_norm': '0.4483', 'learning_rate': '1.998e-06', 'epoch': '1.471'} + 74%|███████▎ | 690/938 [47:54<16:02, 3.88s/it] 74%|███████▎ | 691/938 [47:58<16:26, 3.99s/it] 74%|███████▍ | 692/938 [48:02<16:10, 3.94s/it] 74%|███████▍ | 693/938 [48:06<16:01, 3.92s/it] 74%|███████▍ | 694/938 [48:10<15:36, 3.84s/it] 74%|███████▍ | 695/938 [48:14<15:35, 3.85s/it] 74%|███████▍ | 696/938 [48:18<15:40, 3.89s/it] 74%|███████▍ | 697/938 [48:22<15:58, 3.98s/it] 74%|███████▍ | 698/938 [48:26<16:08, 4.03s/it] 75%|███████▍ | 699/938 [48:30<16:25, 4.12s/it] 75%|███████▍ | 700/938 [48:35<17:15, 4.35s/it] {'loss': '0.2778', 'grad_norm': '0.4514', 'learning_rate': '1.851e-06', 'epoch': '1.493'} + 75%|███████▍ | 700/938 [48:35<17:15, 4.35s/it] 75%|███████▍ | 701/938 [48:39<16:56, 4.29s/it] 75%|███████▍ | 702/938 [48:43<16:04, 4.09s/it] 75%|███████▍ | 703/938 [48:48<16:35, 4.24s/it] 75%|███████▌ | 704/938 [48:53<17:30, 4.49s/it] 75%|███████▌ | 705/938 [48:56<16:23, 4.22s/it] 75%|███████▌ | 706/938 [49:01<16:30, 4.27s/it] 75%|███████▌ | 707/938 [49:05<16:25, 4.26s/it] 75%|███████▌ | 708/938 [49:09<16:37, 4.34s/it] 76%|███████▌ | 709/938 [49:14<16:37, 4.36s/it] 76%|███████▌ | 710/938 [49:18<16:28, 4.33s/it] {'loss': '0.2632', 'grad_norm': '0.4736', 'learning_rate': '1.709e-06', 'epoch': '1.514'} + 76%|███████▌ | 710/938 [49:18<16:28, 4.33s/it] 76%|███████▌ | 711/938 [49:22<16:19, 4.32s/it] 76%|███████▌ | 712/938 [49:26<15:48, 4.19s/it] 76%|███████▌ | 713/938 [49:30<15:14, 4.06s/it] 76%|███████▌ | 714/938 [49:35<15:45, 4.22s/it] 76%|███████▌ | 715/938 [49:38<15:01, 4.04s/it] 76%|███████▋ | 716/938 [49:42<14:33, 3.93s/it] 76%|███████▋ | 717/938 [49:46<14:20, 3.90s/it] 77%|███████▋ | 718/938 [49:49<14:08, 3.85s/it] 77%|███████▋ | 719/938 [49:53<13:49, 3.79s/it] 77%|███████▋ | 720/938 [49:58<14:40, 4.04s/it] {'loss': '0.2699', 'grad_norm': '0.4452', 'learning_rate': '1.571e-06', 'epoch': '1.535'} + 77%|███████▋ | 720/938 [49:58<14:40, 4.04s/it] 77%|███████▋ | 721/938 [50:03<15:28, 4.28s/it] 77%|███████▋ | 722/938 [50:06<14:41, 4.08s/it] 77%|███████▋ | 723/938 [50:11<15:13, 4.25s/it] 77%|███████▋ | 724/938 [50:15<15:15, 4.28s/it] 77%|███████▋ | 725/938 [50:20<15:49, 4.46s/it] 77%|███████▋ | 726/938 [50:24<15:09, 4.29s/it] 78%|███████▊ | 727/938 [50:28<14:46, 4.20s/it] 78%|███████▊ | 728/938 [50:32<14:17, 4.08s/it] 78%|███████▊ | 729/938 [50:36<14:41, 4.22s/it] 78%|███████▊ | 730/938 [50:41<15:25, 4.45s/it] {'loss': '0.2556', 'grad_norm': '0.4535', 'learning_rate': '1.438e-06', 'epoch': '1.557'} + 78%|███████▊ | 730/938 [50:41<15:25, 4.45s/it] 78%|███████▊ | 731/938 [50:46<15:07, 4.38s/it] 78%|███████▊ | 732/938 [50:50<14:44, 4.29s/it] 78%|███████▊ | 733/938 [50:54<14:54, 4.36s/it] 78%|███████▊ | 734/938 [50:58<14:36, 4.30s/it] 78%|███████▊ | 735/938 [51:02<13:51, 4.10s/it] 78%|███████▊ | 736/938 [51:06<13:56, 4.14s/it] 79%|███████▊ | 737/938 [51:10<13:24, 4.00s/it] 79%|███████▊ | 738/938 [51:14<13:35, 4.08s/it] 79%|███████▉ | 739/938 [51:18<13:02, 3.93s/it] 79%|███████▉ | 740/938 [51:22<13:25, 4.07s/it] {'loss': '0.2628', 'grad_norm': '0.4523', 'learning_rate': '1.31e-06', 'epoch': '1.578'} + 79%|███████▉ | 740/938 [51:22<13:25, 4.07s/it] 79%|███████▉ | 741/938 [51:26<13:34, 4.13s/it] 79%|███████▉ | 742/938 [51:30<13:21, 4.09s/it] 79%|███████▉ | 743/938 [51:34<13:10, 4.06s/it] 79%|███████▉ | 744/938 [51:38<12:53, 3.99s/it] 79%|███████▉ | 745/938 [51:42<12:46, 3.97s/it] 80%|███████▉ | 746/938 [51:46<12:22, 3.87s/it] 80%|███████▉ | 747/938 [51:49<12:06, 3.80s/it] 80%|███████▉ | 748/938 [51:53<12:09, 3.84s/it] 80%|███████▉ | 749/938 [51:57<11:41, 3.71s/it] 80%|███████▉ | 750/938 [52:02<12:46, 4.08s/it] {'loss': '0.2727', 'grad_norm': '0.4335', 'learning_rate': '1.187e-06', 'epoch': '1.599'} + 80%|███████▉ | 750/938 [52:02<12:46, 4.08s/it] 80%|████████ | 751/938 [52:06<13:06, 4.21s/it] 80%|████████ | 752/938 [52:10<12:57, 4.18s/it] 80%|████████ | 753/938 [52:14<12:39, 4.10s/it] 80%|████████ | 754/938 [52:19<13:10, 4.30s/it] 80%|████████ | 755/938 [52:24<13:26, 4.41s/it] 81%|████████ | 756/938 [52:27<12:37, 4.16s/it] 81%|████████ | 757/938 [52:32<13:26, 4.46s/it] 81%|████████ | 758/938 [52:36<13:00, 4.34s/it] 81%|████████ | 759/938 [52:41<12:55, 4.33s/it] 81%|████████ | 760/938 [52:45<12:51, 4.34s/it] {'loss': '0.2753', 'grad_norm': '0.4579', 'learning_rate': '1.069e-06', 'epoch': '1.621'} + 81%|████████ | 760/938 [52:45<12:51, 4.34s/it] 81%|████████ | 761/938 [52:49<12:26, 4.22s/it] 81%|████████ | 762/938 [52:53<12:20, 4.21s/it] 81%|████████▏ | 763/938 [52:57<11:59, 4.11s/it] 81%|████████▏ | 764/938 [53:01<11:59, 4.13s/it] 82%|████████▏ | 765/938 [53:05<11:49, 4.10s/it] 82%|████████▏ | 766/938 [53:09<11:52, 4.14s/it] 82%|████████▏ | 767/938 [53:15<13:16, 4.66s/it] 82%|████████▏ | 768/938 [53:19<12:42, 4.49s/it] 82%|████████▏ | 769/938 [53:23<12:11, 4.33s/it] 82%|████████▏ | 770/938 [53:28<11:56, 4.27s/it] {'loss': '0.258', 'grad_norm': '0.4879', 'learning_rate': '9.571e-07', 'epoch': '1.642'} + 82%|████████▏ | 770/938 [53:28<11:56, 4.27s/it] 82%|████████▏ | 771/938 [53:32<12:10, 4.37s/it] 82%|████████▏ | 772/938 [53:36<11:49, 4.28s/it] 82%|████████▏ | 773/938 [53:40<11:13, 4.08s/it] 83%|████████▎ | 774/938 [53:44<11:29, 4.21s/it] 83%|████████▎ | 775/938 [53:48<11:17, 4.15s/it] 83%|████████▎ | 776/938 [53:52<11:00, 4.08s/it] 83%|████████▎ | 777/938 [53:57<11:13, 4.18s/it] 83%|████████▎ | 778/938 [54:02<11:47, 4.42s/it] 83%|████████▎ | 779/938 [54:06<11:53, 4.49s/it] 83%|████████▎ | 780/938 [54:10<11:26, 4.34s/it] {'loss': '0.2685', 'grad_norm': '0.4277', 'learning_rate': '8.504e-07', 'epoch': '1.663'} + 83%|████████▎ | 780/938 [54:10<11:26, 4.34s/it] 83%|████████▎ | 781/938 [54:15<11:38, 4.45s/it] 83%|████████▎ | 782/938 [54:19<11:13, 4.32s/it] 83%|████████▎ | 783/938 [54:23<11:00, 4.26s/it] 84%|████████▎ | 784/938 [54:27<10:50, 4.23s/it] 84%|████████▎ | 785/938 [54:32<11:28, 4.50s/it] 84%|████████▍ | 786/938 [54:36<10:56, 4.32s/it] 84%|████████▍ | 787/938 [54:40<10:12, 4.05s/it] 84%|████████▍ | 788/938 [54:44<10:23, 4.15s/it] 84%|████████▍ | 789/938 [54:48<10:06, 4.07s/it] 84%|████████▍ | 790/938 [54:52<10:03, 4.08s/it] {'loss': '0.266', 'grad_norm': '0.4804', 'learning_rate': '7.495e-07', 'epoch': '1.685'} + 84%|████████▍ | 790/938 [54:52<10:03, 4.08s/it] 84%|████████▍ | 791/938 [54:56<09:48, 4.01s/it] 84%|████████▍ | 792/938 [55:00<09:55, 4.08s/it] 85%|████████▍ | 793/938 [55:04<09:38, 3.99s/it] 85%|████████▍ | 794/938 [55:09<10:15, 4.27s/it] 85%|████████▍ | 795/938 [55:13<09:50, 4.13s/it] 85%|████████▍ | 796/938 [55:18<10:16, 4.34s/it] 85%|████████▍ | 797/938 [55:21<09:48, 4.17s/it] 85%|████████▌ | 798/938 [55:26<09:52, 4.23s/it] 85%|████████▌ | 799/938 [55:30<09:48, 4.23s/it] 85%|████████▌ | 800/938 [55:34<09:39, 4.20s/it] {'loss': '0.2572', 'grad_norm': '0.4831', 'learning_rate': '6.544e-07', 'epoch': '1.706'} + 85%|████████▌ | 800/938 [55:34<09:39, 4.20s/it] 85%|████████▌ | 801/938 [55:38<09:30, 4.16s/it] 86%|████████▌ | 802/938 [55:42<09:27, 4.18s/it] 86%|████████▌ | 803/938 [55:47<09:27, 4.21s/it] 86%|████████▌ | 804/938 [55:50<09:08, 4.10s/it] 86%|████████▌ | 805/938 [55:54<08:47, 3.96s/it] 86%|████████▌ | 806/938 [55:59<09:01, 4.10s/it] 86%|████████▌ | 807/938 [56:03<09:08, 4.19s/it] 86%|████████▌ | 808/938 [56:08<09:25, 4.35s/it] 86%|████████▌ | 809/938 [56:12<09:12, 4.28s/it] 86%|████████▋ | 810/938 [56:15<08:38, 4.05s/it] {'loss': '0.2663', 'grad_norm': '0.5086', 'learning_rate': '5.654e-07', 'epoch': '1.727'} + 86%|████████▋ | 810/938 [56:15<08:38, 4.05s/it] 86%|████████▋ | 811/938 [56:20<08:46, 4.15s/it] 87%|████████▋ | 812/938 [56:24<08:59, 4.28s/it] 87%|████████▋ | 813/938 [56:28<08:51, 4.25s/it] 87%|████████▋ | 814/938 [56:33<08:52, 4.29s/it] 87%|████████▋ | 815/938 [56:37<08:38, 4.22s/it] 87%|████████▋ | 816/938 [56:41<08:43, 4.29s/it] 87%|████████▋ | 817/938 [56:46<08:42, 4.32s/it] 87%|████████▋ | 818/938 [56:51<09:20, 4.67s/it] 87%|████████▋ | 819/938 [56:56<09:12, 4.64s/it] 87%|████████▋ | 820/938 [57:00<08:35, 4.37s/it] {'loss': '0.2616', 'grad_norm': '0.478', 'learning_rate': '4.825e-07', 'epoch': '1.749'} + 87%|████████▋ | 820/938 [57:00<08:35, 4.37s/it] 88%|████████▊ | 821/938 [57:05<08:53, 4.56s/it] 88%|████████▊ | 822/938 [57:08<08:05, 4.19s/it] 88%|████████▊ | 823/938 [57:12<07:51, 4.10s/it] 88%|████████▊ | 824/938 [57:16<07:42, 4.06s/it] 88%|████████▊ | 825/938 [57:20<07:34, 4.02s/it] 88%|████████▊ | 826/938 [57:24<07:33, 4.05s/it] 88%|████████▊ | 827/938 [57:28<07:27, 4.03s/it] 88%|████████▊ | 828/938 [57:32<07:31, 4.10s/it] 88%|████████▊ | 829/938 [57:36<07:16, 4.00s/it] 88%|████████▊ | 830/938 [57:40<07:15, 4.03s/it] {'loss': '0.2693', 'grad_norm': '0.4145', 'learning_rate': '4.059e-07', 'epoch': '1.77'} + 88%|████████▊ | 830/938 [57:40<07:15, 4.03s/it] 89%|████████▊ | 831/938 [57:44<07:20, 4.11s/it] 89%|████████▊ | 832/938 [57:48<06:56, 3.93s/it] 89%|████████▉ | 833/938 [57:52<07:15, 4.15s/it] 89%|████████▉ | 834/938 [57:56<07:01, 4.05s/it] 89%|████████▉ | 835/938 [58:01<07:07, 4.15s/it] 89%|████████▉ | 836/938 [58:05<07:13, 4.25s/it] 89%|████████▉ | 837/938 [58:09<07:04, 4.20s/it] 89%|████████▉ | 838/938 [58:13<06:45, 4.05s/it] 89%|████████▉ | 839/938 [58:17<06:43, 4.08s/it] 90%|████████▉ | 840/938 [58:21<06:48, 4.17s/it] {'loss': '0.2727', 'grad_norm': '0.4312', 'learning_rate': '3.357e-07', 'epoch': '1.791'} + 90%|████████▉ | 840/938 [58:21<06:48, 4.17s/it] 90%|████████▉ | 841/938 [58:25<06:36, 4.08s/it] 90%|████████▉ | 842/938 [58:30<06:53, 4.31s/it] 90%|████████▉ | 843/938 [58:34<06:36, 4.18s/it] 90%|████████▉ | 844/938 [58:38<06:27, 4.12s/it] 90%|█████████ | 845/938 [58:42<06:24, 4.14s/it] 90%|█████████ | 846/938 [58:46<06:08, 4.00s/it] 90%|█████████ | 847/938 [58:50<06:16, 4.14s/it] 90%|█████████ | 848/938 [58:54<06:14, 4.16s/it] 91%|█████████ | 849/938 [58:58<05:56, 4.00s/it] 91%|█████████ | 850/938 [59:02<05:47, 3.95s/it] {'loss': '0.276', 'grad_norm': '0.465', 'learning_rate': '2.719e-07', 'epoch': '1.813'} + 91%|█████████ | 850/938 [59:02<05:47, 3.95s/it] 91%|█████████ | 851/938 [59:06<05:51, 4.04s/it] 91%|█████████ | 852/938 [59:11<05:56, 4.14s/it] 91%|█████████ | 853/938 [59:15<06:01, 4.26s/it] 91%|█████████ | 854/938 [59:19<05:51, 4.19s/it] 91%|█████████ | 855/938 [59:23<05:42, 4.12s/it] 91%|█████████▏| 856/938 [59:27<05:27, 3.99s/it] 91%|█████████▏| 857/938 [59:31<05:32, 4.10s/it] 91%|█████████▏| 858/938 [59:35<05:23, 4.04s/it] 92%|█████████▏| 859/938 [59:39<05:10, 3.93s/it] 92%|█████████▏| 860/938 [59:43<05:08, 3.96s/it] {'loss': '0.2737', 'grad_norm': '0.4531', 'learning_rate': '2.146e-07', 'epoch': '1.834'} + 92%|█████████▏| 860/938 [59:43<05:08, 3.96s/it] 92%|█████████▏| 861/938 [59:47<05:19, 4.14s/it] 92%|█████████▏| 862/938 [59:51<05:13, 4.13s/it] 92%|█████████▏| 863/938 [59:56<05:27, 4.37s/it] 92%|█████████▏| 864/938 [1:00:00<05:16, 4.28s/it] 92%|█████████▏| 865/938 [1:00:04<05:08, 4.22s/it] 92%|█████████▏| 866/938 [1:00:09<05:02, 4.20s/it] 92%|█████████▏| 867/938 [1:00:13<05:02, 4.26s/it] 93%|█████████▎| 868/938 [1:00:18<05:19, 4.57s/it] 93%|█████████▎| 869/938 [1:00:23<05:21, 4.65s/it] 93%|█████████▎| 870/938 [1:00:28<05:18, 4.69s/it] {'loss': '0.2506', 'grad_norm': '0.4372', 'learning_rate': '1.64e-07', 'epoch': '1.855'} + 93%|█████████▎| 870/938 [1:00:28<05:18, 4.69s/it] 93%|█████████▎| 871/938 [1:00:32<05:06, 4.57s/it] 93%|█████████▎| 872/938 [1:00:36<04:51, 4.41s/it] 93%|█████████▎| 873/938 [1:00:41<04:44, 4.38s/it] 93%|█████████▎| 874/938 [1:00:45<04:34, 4.28s/it] 93%|█████████▎| 875/938 [1:00:49<04:24, 4.19s/it] 93%|█████████▎| 876/938 [1:00:53<04:14, 4.11s/it] 93%|█████████▎| 877/938 [1:00:57<04:17, 4.23s/it] 94%|█████████▎| 878/938 [1:01:01<04:01, 4.03s/it] 94%|█████████▎| 879/938 [1:01:04<03:51, 3.92s/it] 94%|█████████▍| 880/938 [1:01:08<03:46, 3.90s/it] {'loss': '0.2735', 'grad_norm': '0.464', 'learning_rate': '1.201e-07', 'epoch': '1.877'} + 94%|█████████▍| 880/938 [1:01:08<03:46, 3.90s/it] 94%|█████████▍| 881/938 [1:01:12<03:45, 3.95s/it] 94%|█████████▍| 882/938 [1:01:16<03:39, 3.92s/it] 94%|█████████▍| 883/938 [1:01:20<03:40, 4.01s/it] 94%|█████████▍| 884/938 [1:01:25<03:42, 4.12s/it] 94%|█████████▍| 885/938 [1:01:29<03:37, 4.10s/it] 94%|█████████▍| 886/938 [1:01:33<03:43, 4.30s/it] 95%|█████████▍| 887/938 [1:01:38<03:40, 4.32s/it] 95%|█████████▍| 888/938 [1:01:41<03:24, 4.09s/it] 95%|█████████▍| 889/938 [1:01:45<03:18, 4.06s/it] 95%|█████████▍| 890/938 [1:01:49<03:11, 3.99s/it] {'loss': '0.2667', 'grad_norm': '0.4555', 'learning_rate': '8.294e-08', 'epoch': '1.898'} + 95%|█████████▍| 890/938 [1:01:49<03:11, 3.99s/it] 95%|█████████▍| 891/938 [1:01:54<03:19, 4.24s/it] 95%|█████████▌| 892/938 [1:01:58<03:05, 4.02s/it] 95%|█████████▌| 893/938 [1:02:01<02:52, 3.84s/it] 95%|█████████▌| 894/938 [1:02:05<02:55, 3.98s/it] 95%|█████████▌| 895/938 [1:02:09<02:47, 3.89s/it] 96%|█████████▌| 896/938 [1:02:13<02:49, 4.04s/it] 96%|█████████▌| 897/938 [1:02:18<02:51, 4.19s/it] 96%|█████████▌| 898/938 [1:02:22<02:46, 4.16s/it] 96%|█████████▌| 899/938 [1:02:27<02:50, 4.36s/it] 96%|█████████▌| 900/938 [1:02:31<02:41, 4.26s/it] {'loss': '0.2759', 'grad_norm': '0.4674', 'learning_rate': '5.259e-08', 'epoch': '1.919'} + 96%|█████████▌| 900/938 [1:02:31<02:41, 4.26s/it] 96%|█████████▌| 901/938 [1:02:35<02:36, 4.22s/it] 96%|█████████▌| 902/938 [1:02:39<02:27, 4.11s/it] 96%|█████████▋| 903/938 [1:02:43<02:22, 4.07s/it] 96%|█████████▋| 904/938 [1:02:46<02:13, 3.93s/it] 96%|█████████▋| 905/938 [1:02:51<02:13, 4.06s/it] 97%|█████████▋| 906/938 [1:02:55<02:11, 4.12s/it] 97%|█████████▋| 907/938 [1:02:59<02:04, 4.01s/it] 97%|█████████▋| 908/938 [1:03:03<01:59, 3.97s/it] 97%|█████████▋| 909/938 [1:03:07<02:00, 4.16s/it] 97%|█████████▋| 910/938 [1:03:12<02:00, 4.32s/it] {'loss': '0.2717', 'grad_norm': '0.464', 'learning_rate': '2.91e-08', 'epoch': '1.941'} + 97%|█████████▋| 910/938 [1:03:12<02:00, 4.32s/it] 97%|█████████▋| 911/938 [1:03:15<01:50, 4.08s/it] 97%|█████████▋| 912/938 [1:03:19<01:43, 4.00s/it] 97%|█████████▋| 913/938 [1:03:23<01:37, 3.91s/it] 97%|█████████▋| 914/938 [1:03:27<01:31, 3.82s/it] 98%|█████████▊| 915/938 [1:03:30<01:28, 3.85s/it] 98%|█████████▊| 916/938 [1:03:34<01:24, 3.85s/it] 98%|█████████▊| 917/938 [1:03:38<01:17, 3.71s/it] 98%|█████████▊| 918/938 [1:03:42<01:17, 3.87s/it] 98%|█████████▊| 919/938 [1:03:46<01:14, 3.93s/it] 98%|█████████▊| 920/938 [1:03:50<01:13, 4.09s/it] {'loss': '0.2673', 'grad_norm': '0.4542', 'learning_rate': '1.25e-08', 'epoch': '1.962'} + 98%|█████████▊| 920/938 [1:03:50<01:13, 4.09s/it] 98%|█████████▊| 921/938 [1:03:54<01:08, 4.02s/it] 98%|█████████▊| 922/938 [1:03:59<01:07, 4.22s/it] 98%|█████████▊| 923/938 [1:04:03<01:03, 4.21s/it] 99%|█████████▊| 924/938 [1:04:07<00:59, 4.23s/it] 99%|█████████▊| 925/938 [1:04:12<00:55, 4.26s/it] 99%|█████████▊| 926/938 [1:04:16<00:50, 4.24s/it] 99%|█████████▉| 927/938 [1:04:19<00:44, 4.01s/it] 99%|█████████▉| 928/938 [1:04:24<00:40, 4.06s/it] 99%|█████████▉| 929/938 [1:04:27<00:35, 3.99s/it] 99%|█████████▉| 930/938 [1:04:31<00:31, 3.99s/it] {'loss': '0.2772', 'grad_norm': '0.479', 'learning_rate': '2.805e-09', 'epoch': '1.983'} + 99%|█████████▉| 930/938 [1:04:31<00:31, 3.99s/it] 99%|█████████▉| 931/938 [1:04:35<00:27, 3.99s/it] 99%|█████████▉| 932/938 [1:04:40<00:25, 4.22s/it] 99%|█████████▉| 933/938 [1:04:45<00:21, 4.35s/it] 100%|█████████▉| 934/938 [1:04:49<00:17, 4.27s/it] 100%|█████████▉| 935/938 [1:04:53<00:12, 4.15s/it] 100%|█████████▉| 936/938 [1:04:57<00:08, 4.17s/it] 100%|█████████▉| 937/938 [1:05:01<00:04, 4.15s/it] 100%|██████████| 938/938 [1:05:04<00:00, 3.74s/it][INFO|trainer.py:3797] 2026-04-29 16:49:43,034 >> Saving model checkpoint to saves/qwen3_1.7b/math/fft/checkpoint-938 +[INFO|configuration_utils.py:432] 2026-04-29 16:49:43,043 >> Configuration saved in saves/qwen3_1.7b/math/fft/checkpoint-938/config.json +[INFO|configuration_utils.py:803] 2026-04-29 16:49:43,046 >> Configuration saved in saves/qwen3_1.7b/math/fft/checkpoint-938/generation_config.json + + Writing model shards: 0%| | 0/1 [00:00> Model weights saved in saves/qwen3_1.7b/math/fft/checkpoint-938/model.safetensors +[INFO|tokenization_utils_base.py:3224] 2026-04-29 16:49:49,361 >> chat template saved in saves/qwen3_1.7b/math/fft/checkpoint-938/chat_template.jinja +[INFO|tokenization_utils_base.py:2078] 2026-04-29 16:49:49,365 >> tokenizer config file saved in saves/qwen3_1.7b/math/fft/checkpoint-938/tokenizer_config.json +[INFO|trainer.py:1863] 2026-04-29 16:49:49,592 >> + +Training completed. Do not forget to share your model on huggingface.co/models =) + + + {'train_runtime': '3915', 'train_samples_per_second': '30.65', 'train_steps_per_second': '0.24', 'train_loss': '0.2964', 'epoch': '2'} + 100%|██████████| 938/938 [1:05:13<00:00, 3.74s/it] 100%|██████████| 938/938 [1:05:13<00:00, 4.17s/it] +[INFO|trainer.py:3797] 2026-04-29 16:49:51,371 >> Saving model checkpoint to saves/qwen3_1.7b/math/fft +[INFO|configuration_utils.py:432] 2026-04-29 16:49:51,380 >> Configuration saved in saves/qwen3_1.7b/math/fft/config.json +[INFO|configuration_utils.py:803] 2026-04-29 16:49:51,384 >> Configuration saved in saves/qwen3_1.7b/math/fft/generation_config.json + Writing model shards: 0%| | 0/1 [00:00> Model weights saved in saves/qwen3_1.7b/math/fft/model.safetensors +[INFO|tokenization_utils_base.py:3224] 2026-04-29 16:49:57,832 >> chat template saved in saves/qwen3_1.7b/math/fft/chat_template.jinja +[INFO|tokenization_utils_base.py:2078] 2026-04-29 16:49:57,836 >> tokenizer config file saved in saves/qwen3_1.7b/math/fft/tokenizer_config.json +ywang29-p4d-debug-worker-0:2470949:2470949 [2] NCCL INFO comm 0x56028de3b5a0 rank 2 nranks 8 cudaDev 2 busId 201c0 - Destroy COMPLETE +ywang29-p4d-debug-worker-0:2470951:2470951 [3] NCCL INFO comm 0x5612df7ca6e0 rank 3 nranks 8 cudaDev 3 busId 201d0 - Destroy COMPLETE +ywang29-p4d-debug-worker-0:2470953:2470953 [5] NCCL INFO comm 0x55c02c235cf0 rank 5 nranks 8 cudaDev 5 busId 901d0 - Destroy COMPLETE +ywang29-p4d-debug-worker-0:2470952:2470952 [4] NCCL INFO comm 0x5647135d02c0 rank 4 nranks 8 cudaDev 4 busId 901c0 - Destroy COMPLETE +ywang29-p4d-debug-worker-0:2470954:2470954 [6] NCCL INFO comm 0x560722415b60 rank 6 nranks 8 cudaDev 6 busId a01c0 - Destroy COMPLETE +ywang29-p4d-debug-worker-0:2470953:2470953 [5] NCCL INFO comm 0x55c0201d0a60 rank 5 nranks 8 cudaDev 5 busId 901d0 - Destroy COMPLETE +ywang29-p4d-debug-worker-0:2470952:2470952 [4] NCCL INFO comm 0x5647078795e0 rank 4 nranks 8 cudaDev 4 busId 901c0 - Destroy COMPLETE +ywang29-p4d-debug-worker-0:2470951:2470951 [3] NCCL INFO comm 0x5612d38dcbf0 rank 3 nranks 8 cudaDev 3 busId 201d0 - Destroy COMPLETE +[INFO|trainer.py:3797] 2026-04-29 16:50:00,902 >> Saving model checkpoint to saves/qwen3_1.7b/math/fft +[INFO|configuration_utils.py:432] 2026-04-29 16:50:00,911 >> Configuration saved in saves/qwen3_1.7b/math/fft/config.json +[INFO|configuration_utils.py:803] 2026-04-29 16:50:00,913 >> Configuration saved in saves/qwen3_1.7b/math/fft/generation_config.json + Writing model shards: 0%| | 0/1 [00:00> Model weights saved in saves/qwen3_1.7b/math/fft/model.safetensors +[INFO|tokenization_utils_base.py:3224] 2026-04-29 16:50:07,499 >> chat template saved in saves/qwen3_1.7b/math/fft/chat_template.jinja +[INFO|tokenization_utils_base.py:2078] 2026-04-29 16:50:07,503 >> tokenizer config file saved in saves/qwen3_1.7b/math/fft/tokenizer_config.json +[INFO|modelcard.py:266] 2026-04-29 16:50:07,981 >> Dropping the following result as it does not have all the necessary fields: +{'task': {'name': 'Causal Language Modeling', 'type': 'text-generation'}} + Processing Files (0 / 0) : | | 0.00B / 0.00B + New Data Upload : | | 0.00B / 0.00B  + + ...b/math/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + ...b/math/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  Processing Files (1 / 1) : 0%| | 11.4MB / 4.07GB, 57.0MB/s + + ...b/math/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + ...b/math/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + ...b/math/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ath/fft/model.safetensors: 0%| | 29.6kB / 4.06GB  + + ...b/math/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ath/fft/model.safetensors: 0%| | 29.6kB / 4.06GB  Processing Files (1 / 2) : 0%| | 11.5MB / 4.07GB, 11.5MB/s + + ...b/math/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ath/fft/model.safetensors: 0%| | 29.6kB / 4.06GB  + + ...b/math/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ath/fft/model.safetensors: 0%| | 29.6kB / 4.06GB  + + ...b/math/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ath/fft/model.safetensors: 0%| | 636kB / 4.06GB  Processing Files (1 / 2) : 0%| | 12.1MB / 4.07GB, 7.54MB/s + New Data Upload : 0%| | 607kB / 134MB, 379kB/s  + + ...b/math/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ath/fft/model.safetensors: 0%| | 1.24MB / 4.06GB  Processing Files (1 / 2) : 0%| | 12.7MB / 4.07GB, 7.04MB/s + New Data Upload : 1%| | 1.21MB / 134MB, 674kB/s  + + ...b/math/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ath/fft/model.safetensors: 0%| | 7.91MB / 4.06GB  Processing Files (1 / 2) : 0%| | 19.3MB / 4.07GB, 9.67MB/s + New Data Upload : 6%|▌ | 7.88MB / 134MB, 3.94MB/s  + + ...b/math/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ath/fft/model.safetensors: 1%| | 21.3MB / 4.06GB  Processing Files (1 / 2) : 1%| | 32.7MB / 4.07GB, 14.9MB/s + New Data Upload : 16%|█▌ | 21.2MB / 134MB, 9.65MB/s  + + ...b/math/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ath/fft/model.safetensors: 1%| | 33.4MB / 4.06GB  Processing Files (1 / 2) : 1%| | 44.8MB / 4.07GB, 18.7MB/s + New Data Upload : 25%|██▍ | 33.4MB / 134MB, 13.9MB/s  + + ...b/math/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ath/fft/model.safetensors: 1%| | 46.1MB / 4.06GB  Processing Files (1 / 2) : 1%|▏ | 57.5MB / 4.07GB, 22.1MB/s + New Data Upload : 34%|███▍ | 46.1MB / 134MB, 17.7MB/s  + + ...b/math/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ath/fft/model.safetensors: 1%|▏ | 58.3MB / 4.06GB  Processing Files (1 / 2) : 2%|▏ | 69.7MB / 4.07GB, 24.9MB/s + New Data Upload : 43%|████▎ | 58.2MB / 134MB, 20.8MB/s  + + ...b/math/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ath/fft/model.safetensors: 2%|▏ | 66.7MB / 4.06GB  Processing Files (1 / 2) : 2%|▏ | 78.2MB / 4.07GB, 26.1MB/s + New Data Upload : 50%|████▉ | 66.7MB / 134MB, 22.2MB/s  + + ...b/math/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ath/fft/model.safetensors: 2%|▏ | 66.7MB / 4.06GB  + + ...b/math/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ath/fft/model.safetensors: 2%|▏ | 66.7MB / 4.06GB  + + ...b/math/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ath/fft/model.safetensors: 2%|▏ | 66.7MB / 4.06GB  + + ...b/math/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ath/fft/model.safetensors: 2%|▏ | 67.1MB / 4.06GB  Processing Files (1 / 2) : 2%|▏ | 78.5MB / 4.07GB, 20.7MB/s + New Data Upload : 50%|████▉ | 67.1MB / 134MB, 17.7MB/s  + + ...b/math/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ath/fft/model.safetensors: 2%|▏ | 67.7MB / 4.06GB  Processing Files (1 / 2) : 2%|▏ | 79.1MB / 4.07GB, 19.8MB/s + New Data Upload : 50%|█████ | 67.7MB / 134MB, 16.9MB/s  + + ...b/math/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ath/fft/model.safetensors: 2%|▏ | 71.4MB / 4.06GB  Processing Files (1 / 2) : 2%|▏ | 82.8MB / 4.07GB, 19.7MB/s + New Data Upload : 35%|███▌ | 71.3MB / 201MB, 17.0MB/s  + + ...b/math/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ath/fft/model.safetensors: 2%|▏ | 78.0MB / 4.06GB  Processing Files (1 / 2) : 2%|▏ | 89.4MB / 4.07GB, 20.3MB/s + New Data Upload : 39%|███▉ | 78.0MB / 201MB, 17.7MB/s  + + ...b/math/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ath/fft/model.safetensors: 2%|▏ | 98.0MB / 4.06GB  Processing Files (1 / 2) : 3%|▎ | 109MB / 4.07GB, 23.8MB/s + New Data Upload : 37%|███▋ | 98.0MB / 268MB, 21.3MB/s  + + ...b/math/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ath/fft/model.safetensors: 3%|▎ | 122MB / 4.06GB  Processing Files (1 / 2) : 3%|▎ | 133MB / 4.07GB, 27.7MB/s + New Data Upload : 45%|████▌ | 122MB / 268MB, 25.3MB/s  + + ...b/math/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ath/fft/model.safetensors: 4%|▎ | 146MB / 4.06GB  Processing Files (1 / 2) : 4%|▍ | 157MB / 4.07GB, 31.5MB/s + New Data Upload : 43%|████▎ | 146MB / 335MB, 29.2MB/s  + + ...b/math/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ath/fft/model.safetensors: 4%|▍ | 173MB / 4.06GB  Processing Files (1 / 2) : 5%|▍ | 185MB / 4.07GB, 35.5MB/s + New Data Upload : 52%|█████▏ | 173MB / 335MB, 33.3MB/s  + + ...b/math/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ath/fft/model.safetensors: 5%|▍ | 194MB / 4.06GB  Processing Files (1 / 2) : 5%|▌ | 206MB / 4.07GB, 38.1MB/s + New Data Upload : 58%|█████▊ | 194MB / 335MB, 36.0MB/s  + + ...b/math/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ath/fft/model.safetensors: 6%|▌ | 225MB / 4.06GB  Processing Files (1 / 2) : 6%|▌ | 236MB / 4.07GB, 42.2MB/s + New Data Upload : 56%|█████▌ | 225MB / 403MB, 40.1MB/s  + + ...b/math/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ath/fft/model.safetensors: 6%|▌ | 251MB / 4.06GB  Processing Files (1 / 2) : 6%|▋ | 263MB / 4.07GB, 45.3MB/s + New Data Upload : 62%|██████▏ | 251MB / 403MB, 43.3MB/s  + + ...b/math/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ath/fft/model.safetensors: 7%|▋ | 280MB / 4.06GB  Processing Files (1 / 2) : 7%|▋ | 292MB / 4.07GB, 48.6MB/s + New Data Upload : 70%|██████▉ | 280MB / 403MB, 46.7MB/s  + + ...b/math/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ath/fft/model.safetensors: 7%|▋ | 304MB / 4.06GB  Processing Files (1 / 2) : 8%|▊ | 315MB / 4.07GB, 50.9MB/s + New Data Upload : 65%|██████▍ | 304MB / 470MB, 49.0MB/s  + + ...b/math/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ath/fft/model.safetensors: 8%|▊ | 333MB / 4.06GB  Processing Files (1 / 2) : 8%|▊ | 345MB / 4.07GB, 53.9MB/s + New Data Upload : 71%|███████ | 333MB / 470MB, 52.1MB/s  + + ...b/math/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ath/fft/model.safetensors: 9%|▉ | 356MB / 4.06GB  Processing Files (1 / 2) : 9%|▉ | 368MB / 4.07GB, 55.7MB/s + New Data Upload : 66%|██████▋ | 356MB / 537MB, 54.0MB/s  + + ...b/math/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ath/fft/model.safetensors: 9%|▉ | 377MB / 4.06GB  Processing Files (1 / 2) : 10%|▉ | 389MB / 4.07GB, 57.1MB/s + New Data Upload : 70%|███████ | 377MB / 537MB, 55.5MB/s  + + ...b/math/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ath/fft/model.safetensors: 10%|█ | 407MB / 4.06GB  Processing Files (1 / 2) : 10%|█ | 418MB / 4.07GB, 59.8MB/s + New Data Upload : 67%|██████▋ | 407MB / 604MB, 58.1MB/s  + + ...b/math/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ath/fft/model.safetensors: 11%|█ | 447MB / 4.06GB  Processing Files (1 / 2) : 11%|█ | 458MB / 4.07GB, 63.7MB/s + New Data Upload : 74%|███████▍ | 447MB / 604MB, 62.1MB/s  + + ...b/math/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ath/fft/model.safetensors: 12%|█▏ | 480MB / 4.06GB  Processing Files (1 / 2) : 12%|█▏ | 492MB / 4.07GB, 66.4MB/s + New Data Upload : 72%|███████▏ | 480MB / 671MB, 64.9MB/s  + + ...b/math/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ath/fft/model.safetensors: 13%|█▎ | 516MB / 4.06GB  Processing Files (1 / 2) : 13%|█▎ | 528MB / 4.07GB, 69.5MB/s + New Data Upload : 77%|███████▋ | 516MB / 671MB, 67.9MB/s  + + ...b/math/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ath/fft/model.safetensors: 14%|█▍ | 563MB / 4.06GB  Processing Files (1 / 2) : 14%|█▍ | 575MB / 4.07GB, 73.7MB/s + New Data Upload : 76%|███████▋ | 563MB / 738MB, 72.2MB/s  + + ...b/math/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ath/fft/model.safetensors: 15%|█▍ | 594MB / 4.06GB  Processing Files (1 / 2) : 15%|█▍ | 605MB / 4.07GB, 75.7MB/s + New Data Upload : 81%|████████ | 594MB / 738MB, 74.2MB/s  + + ...b/math/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ath/fft/model.safetensors: 15%|█▌ | 621MB / 4.06GB  Processing Files (1 / 2) : 16%|█▌ | 632MB / 4.07GB, 77.1MB/s + New Data Upload : 77%|███████▋ | 621MB / 805MB, 75.7MB/s  + + ...b/math/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ath/fft/model.safetensors: 16%|█▌ | 650MB / 4.06GB  Processing Files (1 / 2) : 16%|█▌ | 662MB / 4.07GB, 78.8MB/s + New Data Upload : 81%|████████ | 650MB / 805MB, 77.4MB/s  + + ...b/math/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ath/fft/model.safetensors: 17%|█▋ | 690MB / 4.06GB  Processing Files (1 / 2) : 17%|█▋ | 702MB / 4.07GB, 81.6MB/s + New Data Upload : 86%|████████▌ | 690MB / 805MB, 80.3MB/s  + + ...b/math/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ath/fft/model.safetensors: 17%|█▋ | 708MB / 4.06GB  Processing Files (1 / 2) : 18%|█▊ | 720MB / 4.07GB, 81.8MB/s + New Data Upload : 81%|████████ | 708MB / 872MB, 80.5MB/s  + + ...b/math/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ath/fft/model.safetensors: 18%|█▊ | 743MB / 4.06GB  Processing Files (1 / 2) : 19%|█▊ | 754MB / 4.07GB, 83.8MB/s + New Data Upload : 79%|███████▉ | 743MB / 939MB, 82.5MB/s  + + ...b/math/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ath/fft/model.safetensors: 19%|█▉ | 774MB / 4.06GB  Processing Files (1 / 2) : 19%|█▉ | 786MB / 4.07GB, 85.4MB/s + New Data Upload : 82%|████████▏ | 774MB / 939MB, 84.2MB/s  + + ...b/math/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ath/fft/model.safetensors: 20%|█▉ | 811MB / 4.06GB  Processing Files (1 / 2) : 20%|██ | 822MB / 4.07GB, 87.5MB/s + New Data Upload : 81%|████████ | 811MB / 1.01GB, 86.3MB/s  + + ...b/math/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ath/fft/model.safetensors: 21%|██ | 846MB / 4.06GB  Processing Files (1 / 2) : 21%|██ | 857MB / 4.07GB, 89.3MB/s + New Data Upload : 79%|███████▉ | 845MB / 1.07GB, 88.1MB/s  + + ...b/math/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ath/fft/model.safetensors: 22%|██▏ | 894MB / 4.06GB  Processing Files (1 / 2) : 22%|██▏ | 905MB / 4.07GB, 92.4MB/s + New Data Upload : 83%|████████▎ | 894MB / 1.07GB, 91.2MB/s  + + ...b/math/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ath/fft/model.safetensors: 23%|██▎ | 939MB / 4.06GB  Processing Files (1 / 2) : 23%|██▎ | 950MB / 4.07GB, 95.0MB/s + New Data Upload : 82%|████████▏ | 939MB / 1.14GB, 93.9MB/s  + + ...b/math/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ath/fft/model.safetensors: 24%|██▍ | 978MB / 4.06GB  Processing Files (1 / 2) : 24%|██▍ | 990MB / 4.07GB, 97.0MB/s + New Data Upload : 86%|████████▌ | 978MB / 1.14GB, 95.9MB/s  + + ...b/math/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ath/fft/model.safetensors: 25%|██▌ | 1.02GB / 4.06GB  Processing Files (1 / 2) : 25%|██▌ | 1.03GB / 4.07GB, 99.7MB/s + New Data Upload : 89%|████████▉ | 1.02GB / 1.14GB, 99.7MB/s  + + ...b/math/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ath/fft/model.safetensors: 26%|██▌ | 1.05GB / 4.06GB  Processing Files (1 / 2) : 26%|██▌ | 1.06GB / 4.07GB, 103MB/s + New Data Upload : 92%|█████████▏| 1.05GB / 1.14GB, 103MB/s  + + ...b/math/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ath/fft/model.safetensors: 26%|██▌ | 1.06GB / 4.06GB  Processing Files (1 / 2) : 26%|██▋ | 1.07GB / 4.07GB, 104MB/s + New Data Upload : 93%|█████████▎| 1.06GB / 1.14GB, 104MB/s  + + ...b/math/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ath/fft/model.safetensors: 27%|██▋ | 1.08GB / 4.06GB  Processing Files (1 / 2) : 27%|██▋ | 1.09GB / 4.07GB, 106MB/s + New Data Upload : 95%|█████████▍| 1.08GB / 1.14GB, 106MB/s  + + ...b/math/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ath/fft/model.safetensors: 27%|██▋ | 1.09GB / 4.06GB  Processing Files (1 / 2) : 27%|██▋ | 1.10GB / 4.07GB, 107MB/s + New Data Upload : 96%|█████████▌| 1.09GB / 1.14GB, 107MB/s  + + ...b/math/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ath/fft/model.safetensors: 27%|██▋ | 1.11GB / 4.06GB  Processing Files (1 / 2) : 27%|██▋ | 1.12GB / 4.07GB, 108MB/s + New Data Upload : 97%|█████████▋| 1.11GB / 1.14GB, 108MB/s  + + ...b/math/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ath/fft/model.safetensors: 28%|██▊ | 1.12GB / 4.06GB  Processing Files (1 / 2) : 28%|██▊ | 1.13GB / 4.07GB, 110MB/s + New Data Upload : 98%|█████████▊| 1.12GB / 1.14GB, 110MB/s  + + ...b/math/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ath/fft/model.safetensors: 29%|██▉ | 1.18GB / 4.06GB  Processing Files (1 / 2) : 29%|██▉ | 1.19GB / 4.07GB, 116MB/s + New Data Upload : 99%|█████████▉| 1.13GB / 1.14GB, 111MB/s  + + ...b/math/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ath/fft/model.safetensors: 29%|██▉ | 1.19GB / 4.06GB  Processing Files (1 / 2) : 30%|██▉ | 1.21GB / 4.07GB, 117MB/s + New Data Upload : 94%|█████████▍| 1.14GB / 1.21GB, 112MB/s  + + ...b/math/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ath/fft/model.safetensors: 29%|██▉ | 1.19GB / 4.06GB  + + ...b/math/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ath/fft/model.safetensors: 29%|██▉ | 1.20GB / 4.06GB  Processing Files (1 / 2) : 30%|██▉ | 1.21GB / 4.07GB, 115MB/s + New Data Upload : 90%|████████▉ | 1.14GB / 1.27GB, 110MB/s  + + ...b/math/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ath/fft/model.safetensors: 30%|██▉ | 1.20GB / 4.06GB  Processing Files (1 / 2) : 30%|██▉ | 1.21GB / 4.07GB, 115MB/s + New Data Upload : 86%|████████▌ | 1.15GB / 1.34GB, 109MB/s  + + ...b/math/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ath/fft/model.safetensors: 30%|██▉ | 1.21GB / 4.06GB  Processing Files (1 / 2) : 30%|███ | 1.22GB / 4.07GB, 114MB/s + New Data Upload : 86%|████████▋ | 1.16GB / 1.34GB, 109MB/s  + + ...b/math/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ath/fft/model.safetensors: 31%|███ | 1.25GB / 4.06GB  Processing Files (1 / 2) : 31%|███ | 1.26GB / 4.07GB, 116MB/s + New Data Upload : 85%|████████▍ | 1.19GB / 1.41GB, 111MB/s  + + ...b/math/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ath/fft/model.safetensors: 32%|███▏ | 1.29GB / 4.06GB  Processing Files (1 / 2) : 32%|███▏ | 1.30GB / 4.07GB, 120MB/s + New Data Upload : 88%|████████▊ | 1.23GB / 1.41GB, 115MB/s  + + ...b/math/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ath/fft/model.safetensors: 33%|███▎ | 1.32GB / 4.06GB  Processing Files (1 / 2) : 33%|███▎ | 1.33GB / 4.07GB, 123MB/s + New Data Upload : 86%|████████▌ | 1.27GB / 1.48GB, 118MB/s  + + ...b/math/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ath/fft/model.safetensors: 34%|███▎ | 1.37GB / 4.06GB  Processing Files (1 / 2) : 34%|███▍ | 1.38GB / 4.07GB, 128MB/s + New Data Upload : 85%|████████▌ | 1.31GB / 1.54GB, 122MB/s  + + ...b/math/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ath/fft/model.safetensors: 35%|███▍ | 1.40GB / 4.06GB  Processing Files (1 / 2) : 35%|███▍ | 1.42GB / 4.07GB, 131MB/s + New Data Upload : 88%|████████▊ | 1.35GB / 1.54GB, 126MB/s  + + ...b/math/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ath/fft/model.safetensors: 35%|███▌ | 1.44GB / 4.06GB  Processing Files (1 / 2) : 36%|███▌ | 1.45GB / 4.07GB, 134MB/s + New Data Upload : 86%|████████▌ | 1.38GB / 1.61GB, 129MB/s  + + ...b/math/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ath/fft/model.safetensors: 36%|███▌ | 1.46GB / 4.06GB  Processing Files (1 / 2) : 36%|███▌ | 1.48GB / 4.07GB, 137MB/s + New Data Upload : 88%|████████▊ | 1.41GB / 1.61GB, 132MB/s  + + ...b/math/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ath/fft/model.safetensors: 37%|███▋ | 1.49GB / 4.06GB  Processing Files (1 / 2) : 37%|███▋ | 1.50GB / 4.07GB, 139MB/s + New Data Upload : 86%|████████▌ | 1.44GB / 1.68GB, 134MB/s  + + ...b/math/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ath/fft/model.safetensors: 38%|███▊ | 1.53GB / 4.06GB  Processing Files (1 / 2) : 38%|███▊ | 1.54GB / 4.07GB, 142MB/s + New Data Upload : 85%|████████▍ | 1.47GB / 1.74GB, 137MB/s  + + ...b/math/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ath/fft/model.safetensors: 39%|███▉ | 1.58GB / 4.06GB  Processing Files (1 / 2) : 39%|███▉ | 1.59GB / 4.07GB, 145MB/s + New Data Upload : 87%|████████▋ | 1.52GB / 1.74GB, 140MB/s  + + ...b/math/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ath/fft/model.safetensors: 40%|████ | 1.63GB / 4.06GB  Processing Files (1 / 2) : 40%|████ | 1.64GB / 4.07GB, 148MB/s + New Data Upload : 87%|████████▋ | 1.57GB / 1.81GB, 142MB/s  + + ...b/math/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ath/fft/model.safetensors: 42%|████▏ | 1.69GB / 4.06GB  Processing Files (1 / 2) : 42%|████▏ | 1.70GB / 4.07GB, 151MB/s + New Data Upload : 87%|████████▋ | 1.63GB / 1.88GB, 146MB/s  + + ...b/math/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ath/fft/model.safetensors: 43%|████▎ | 1.75GB / 4.06GB  Processing Files (1 / 2) : 43%|████▎ | 1.76GB / 4.07GB, 155MB/s + New Data Upload : 90%|█████████ | 1.70GB / 1.88GB, 149MB/s  + + ...b/math/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ath/fft/model.safetensors: 44%|████▍ | 1.80GB / 4.06GB  Processing Files (1 / 2) : 44%|████▍ | 1.81GB / 4.07GB, 157MB/s + New Data Upload : 90%|████████▉ | 1.75GB / 1.95GB, 152MB/s  + + ...b/math/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ath/fft/model.safetensors: 45%|████▌ | 1.83GB / 4.06GB  Processing Files (1 / 2) : 45%|████▌ | 1.84GB / 4.07GB, 158MB/s + New Data Upload : 91%|█████████▏| 1.78GB / 1.95GB, 152MB/s  + + ...b/math/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ath/fft/model.safetensors: 46%|████▌ | 1.87GB / 4.06GB  Processing Files (1 / 2) : 46%|████▌ | 1.88GB / 4.07GB, 159MB/s + New Data Upload : 90%|█████████ | 1.81GB / 2.01GB, 153MB/s  + + ...b/math/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ath/fft/model.safetensors: 47%|████▋ | 1.91GB / 4.06GB  Processing Files (1 / 2) : 47%|████▋ | 1.92GB / 4.07GB, 160MB/s + New Data Upload : 92%|█████████▏| 1.85GB / 2.01GB, 154MB/s  + + ...b/math/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ath/fft/model.safetensors: 48%|████▊ | 1.94GB / 4.06GB  Processing Files (1 / 2) : 48%|████▊ | 1.95GB / 4.07GB, 160MB/s + New Data Upload : 91%|█████████ | 1.88GB / 2.08GB, 155MB/s  + + ...b/math/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ath/fft/model.safetensors: 49%|████▊ | 1.97GB / 4.06GB  Processing Files (1 / 2) : 49%|████▊ | 1.99GB / 4.07GB, 161MB/s + New Data Upload : 92%|█████████▏| 1.92GB / 2.08GB, 156MB/s  + + ...b/math/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ath/fft/model.safetensors: 50%|████▉ | 2.02GB / 4.06GB  Processing Files (1 / 2) : 50%|████▉ | 2.04GB / 4.07GB, 164MB/s + New Data Upload : 92%|█████████▏| 1.97GB / 2.15GB, 158MB/s  + + ...b/math/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ath/fft/model.safetensors: 51%|█████ | 2.06GB / 4.06GB  Processing Files (1 / 2) : 51%|█████ | 2.07GB / 4.07GB, 165MB/s + New Data Upload : 94%|█████████▎| 2.01GB / 2.15GB, 160MB/s  + + ...b/math/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ath/fft/model.safetensors: 52%|█████▏ | 2.09GB / 4.06GB  Processing Files (1 / 2) : 52%|█████▏ | 2.11GB / 4.07GB, 165MB/s + New Data Upload : 92%|█████████▏| 2.04GB / 2.21GB, 160MB/s  + + ...b/math/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ath/fft/model.safetensors: 52%|█████▏ | 2.13GB / 4.06GB  Processing Files (1 / 2) : 53%|█████▎ | 2.14GB / 4.07GB, 165MB/s + New Data Upload : 94%|█████████▍| 2.08GB / 2.21GB, 160MB/s  + + ...b/math/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ath/fft/model.safetensors: 53%|█████▎ | 2.16GB / 4.06GB  Processing Files (1 / 2) : 53%|█████▎ | 2.17GB / 4.07GB, 164MB/s + New Data Upload : 92%|█████████▏| 2.10GB / 2.28GB, 159MB/s  + + ...b/math/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ath/fft/model.safetensors: 54%|█████▎ | 2.18GB / 4.06GB  Processing Files (1 / 2) : 54%|█████▍ | 2.19GB / 4.07GB, 163MB/s + New Data Upload : 91%|█████████ | 2.13GB / 2.35GB, 158MB/s  + + ...b/math/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ath/fft/model.safetensors: 54%|█████▍ | 2.21GB / 4.06GB  Processing Files (1 / 2) : 54%|█████▍ | 2.22GB / 4.07GB, 161MB/s + New Data Upload : 92%|█████████▏| 2.15GB / 2.35GB, 156MB/s  + + ...b/math/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ath/fft/model.safetensors: 55%|█████▌ | 2.25GB / 4.06GB  Processing Files (1 / 2) : 56%|█████▌ | 2.26GB / 4.07GB, 163MB/s + New Data Upload : 91%|█████████ | 2.20GB / 2.41GB, 157MB/s  + + ...b/math/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ath/fft/model.safetensors: 56%|█████▌ | 2.28GB / 4.06GB  Processing Files (1 / 2) : 56%|█████▋ | 2.30GB / 4.07GB, 163MB/s + New Data Upload : 92%|█████████▏| 2.23GB / 2.41GB, 158MB/s  + + ...b/math/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ath/fft/model.safetensors: 57%|█████▋ | 2.33GB / 4.06GB  Processing Files (1 / 2) : 57%|█████▋ | 2.34GB / 4.07GB, 165MB/s + New Data Upload : 92%|█████████▏| 2.27GB / 2.48GB, 159MB/s  + + ...b/math/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ath/fft/model.safetensors: 58%|█████▊ | 2.37GB / 4.06GB  Processing Files (1 / 2) : 58%|█████▊ | 2.38GB / 4.07GB, 164MB/s + New Data Upload : 93%|█████████▎| 2.31GB / 2.48GB, 159MB/s  + + ...b/math/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ath/fft/model.safetensors: 59%|█████▉ | 2.41GB / 4.06GB  Processing Files (1 / 2) : 59%|█████▉ | 2.42GB / 4.07GB, 166MB/s + New Data Upload : 92%|█████████▏| 2.35GB / 2.55GB, 161MB/s  + + ...b/math/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ath/fft/model.safetensors: 60%|██████ | 2.44GB / 4.06GB  Processing Files (1 / 2) : 60%|██████ | 2.46GB / 4.07GB, 167MB/s + New Data Upload : 91%|█████████▏| 2.39GB / 2.62GB, 161MB/s  + + ...b/math/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ath/fft/model.safetensors: 61%|██████ | 2.49GB / 4.06GB  Processing Files (1 / 2) : 61%|██████▏ | 2.50GB / 4.07GB, 168MB/s + New Data Upload : 93%|█████████▎| 2.43GB / 2.62GB, 162MB/s  + + ...b/math/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ath/fft/model.safetensors: 62%|██████▏ | 2.52GB / 4.06GB  Processing Files (1 / 2) : 62%|██████▏ | 2.53GB / 4.07GB, 168MB/s + New Data Upload : 94%|█████████▍| 2.47GB / 2.62GB, 163MB/s  + + ...b/math/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ath/fft/model.safetensors: 63%|██████▎ | 2.56GB / 4.06GB  Processing Files (1 / 2) : 63%|██████▎ | 2.57GB / 4.07GB, 168MB/s + New Data Upload : 93%|█████████▎| 2.51GB / 2.68GB, 163MB/s  + + ...b/math/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ath/fft/model.safetensors: 64%|██████▍ | 2.60GB / 4.06GB  Processing Files (1 / 2) : 64%|██████▍ | 2.61GB / 4.07GB, 167MB/s + New Data Upload : 92%|█████████▏| 2.54GB / 2.75GB, 162MB/s  + + ...b/math/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ath/fft/model.safetensors: 65%|██████▍ | 2.64GB / 4.06GB  Processing Files (1 / 2) : 65%|██████▍ | 2.65GB / 4.07GB, 166MB/s + New Data Upload : 92%|█████████▏| 2.58GB / 2.82GB, 161MB/s  + + ...b/math/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ath/fft/model.safetensors: 66%|██████▌ | 2.68GB / 4.06GB  Processing Files (1 / 2) : 66%|██████▌ | 2.69GB / 4.07GB, 167MB/s + New Data Upload : 93%|█████████▎| 2.62GB / 2.82GB, 161MB/s  + + ...b/math/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ath/fft/model.safetensors: 67%|██████▋ | 2.72GB / 4.06GB  Processing Files (1 / 2) : 67%|██████▋ | 2.73GB / 4.07GB, 167MB/s + New Data Upload : 95%|█████████▍| 2.67GB / 2.82GB, 162MB/s  + + ...b/math/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ath/fft/model.safetensors: 68%|██████▊ | 2.76GB / 4.06GB  Processing Files (1 / 2) : 68%|██████▊ | 2.77GB / 4.07GB, 168MB/s + New Data Upload : 94%|█████████▍| 2.71GB / 2.88GB, 163MB/s  + + ...b/math/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ath/fft/model.safetensors: 69%|██████▊ | 2.79GB / 4.06GB  Processing Files (1 / 2) : 69%|██████▉ | 2.80GB / 4.07GB, 170MB/s + New Data Upload : 95%|█████████▍| 2.74GB / 2.88GB, 165MB/s  + + ...b/math/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ath/fft/model.safetensors: 69%|██████▉ | 2.82GB / 4.06GB  Processing Files (1 / 2) : 70%|██████▉ | 2.83GB / 4.07GB, 171MB/s + New Data Upload : 94%|█████████▍| 2.77GB / 2.95GB, 166MB/s  + + ...b/math/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ath/fft/model.safetensors: 70%|███████ | 2.86GB / 4.06GB  Processing Files (1 / 2) : 70%|███████ | 2.87GB / 4.07GB, 173MB/s + New Data Upload : 95%|█████████▍| 2.80GB / 2.95GB, 168MB/s  + + ...b/math/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ath/fft/model.safetensors: 71%|███████ | 2.88GB / 4.06GB  Processing Files (1 / 2) : 71%|███████ | 2.89GB / 4.07GB, 174MB/s + New Data Upload : 94%|█████████▎| 2.83GB / 3.02GB, 169MB/s  + + ...b/math/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ath/fft/model.safetensors: 72%|███████▏ | 2.92GB / 4.06GB  Processing Files (1 / 2) : 72%|███████▏ | 2.93GB / 4.07GB, 176MB/s + New Data Upload : 93%|█████████▎| 2.86GB / 3.09GB, 171MB/s  + + ...b/math/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ath/fft/model.safetensors: 73%|███████▎ | 2.95GB / 4.06GB  Processing Files (1 / 2) : 73%|███████▎ | 2.96GB / 4.07GB, 173MB/s + New Data Upload : 94%|█████████▍| 2.90GB / 3.09GB, 173MB/s  + + ...b/math/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ath/fft/model.safetensors: 73%|███████▎ | 2.99GB / 4.06GB  Processing Files (1 / 2) : 74%|███████▎ | 3.00GB / 4.07GB, 176MB/s + New Data Upload : 93%|█████████▎| 2.93GB / 3.15GB, 176MB/s  + + ...b/math/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ath/fft/model.safetensors: 74%|███████▍ | 3.00GB / 4.06GB  Processing Files (1 / 2) : 74%|███████▍ | 3.02GB / 4.07GB, 177MB/s + New Data Upload : 94%|█████████▎| 2.95GB / 3.15GB, 177MB/s  + + ...b/math/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ath/fft/model.safetensors: 75%|███████▍ | 3.05GB / 4.06GB  Processing Files (1 / 2) : 75%|███████▌ | 3.06GB / 4.07GB, 181MB/s + New Data Upload : 93%|█████████▎| 2.99GB / 3.22GB, 181MB/s  + + ...b/math/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ath/fft/model.safetensors: 76%|███████▌ | 3.08GB / 4.06GB  Processing Files (1 / 2) : 76%|███████▌ | 3.09GB / 4.07GB, 184MB/s + New Data Upload : 94%|█████████▍| 3.02GB / 3.22GB, 184MB/s  + + ...b/math/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ath/fft/model.safetensors: 77%|███████▋ | 3.13GB / 4.06GB  Processing Files (1 / 2) : 77%|███████▋ | 3.14GB / 4.07GB, 188MB/s + New Data Upload : 93%|█████████▎| 3.07GB / 3.29GB, 188MB/s  + + ...b/math/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ath/fft/model.safetensors: 78%|███████▊ | 3.16GB / 4.06GB  Processing Files (1 / 2) : 78%|███████▊ | 3.18GB / 4.07GB, 188MB/s + New Data Upload : 95%|█████████▍| 3.11GB / 3.29GB, 188MB/s  + + ...b/math/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ath/fft/model.safetensors: 79%|███████▉ | 3.20GB / 4.06GB  Processing Files (1 / 2) : 79%|███████▉ | 3.21GB / 4.07GB, 188MB/s + New Data Upload : 94%|█████████▍| 3.15GB / 3.35GB, 188MB/s  + + ...b/math/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ath/fft/model.safetensors: 80%|███████▉ | 3.24GB / 4.06GB  Processing Files (1 / 2) : 80%|███████▉ | 3.25GB / 4.07GB, 188MB/s + New Data Upload : 95%|█████████▌| 3.19GB / 3.35GB, 188MB/s  + + ...b/math/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ath/fft/model.safetensors: 81%|████████ | 3.27GB / 4.06GB  Processing Files (1 / 2) : 81%|████████ | 3.28GB / 4.07GB, 187MB/s + New Data Upload : 94%|█████████▍| 3.22GB / 3.42GB, 187MB/s  + + ...b/math/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ath/fft/model.safetensors: 81%|████████▏ | 3.30GB / 4.06GB  Processing Files (1 / 2) : 81%|████████▏ | 3.32GB / 4.07GB, 186MB/s + New Data Upload : 95%|█████████▌| 3.25GB / 3.42GB, 186MB/s  + + ...b/math/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ath/fft/model.safetensors: 82%|████████▏ | 3.35GB / 4.06GB  Processing Files (1 / 2) : 82%|████████▏ | 3.36GB / 4.07GB, 187MB/s + New Data Upload : 94%|█████████▍| 3.29GB / 3.49GB, 187MB/s  + + ...b/math/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ath/fft/model.safetensors: 84%|████████▎ | 3.40GB / 4.06GB  Processing Files (1 / 2) : 84%|████████▎ | 3.41GB / 4.07GB, 189MB/s + New Data Upload : 94%|█████████▍| 3.34GB / 3.55GB, 189MB/s  + + ...b/math/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ath/fft/model.safetensors: 84%|████████▍ | 3.43GB / 4.06GB  Processing Files (1 / 2) : 84%|████████▍ | 3.44GB / 4.07GB, 190MB/s + New Data Upload : 95%|█████████▍| 3.37GB / 3.55GB, 190MB/s  + + ...b/math/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ath/fft/model.safetensors: 85%|████████▌ | 3.47GB / 4.06GB  Processing Files (1 / 2) : 85%|████████▌ | 3.48GB / 4.07GB, 190MB/s + New Data Upload : 94%|█████████▍| 3.42GB / 3.62GB, 190MB/s  + + ...b/math/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ath/fft/model.safetensors: 86%|████████▋ | 3.51GB / 4.06GB  Processing Files (1 / 2) : 87%|████████▋ | 3.53GB / 4.07GB, 190MB/s + New Data Upload : 96%|█████████▌| 3.46GB / 3.62GB, 190MB/s  + + ...b/math/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ath/fft/model.safetensors: 88%|████████▊ | 3.56GB / 4.06GB  Processing Files (1 / 2) : 88%|████████▊ | 3.57GB / 4.07GB, 189MB/s + New Data Upload : 95%|█████████▍| 3.50GB / 3.69GB, 189MB/s  + + ...b/math/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ath/fft/model.safetensors: 88%|████████▊ | 3.58GB / 4.06GB  Processing Files (1 / 2) : 88%|████████▊ | 3.59GB / 4.07GB, 185MB/s + New Data Upload : 96%|█████████▌| 3.52GB / 3.69GB, 185MB/s  + + ...b/math/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ath/fft/model.safetensors: 89%|████████▉ | 3.61GB / 4.06GB  Processing Files (1 / 2) : 89%|████████▉ | 3.62GB / 4.07GB, 182MB/s + New Data Upload : 95%|█████████▍| 3.56GB / 3.76GB, 182MB/s  + + ...b/math/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ath/fft/model.safetensors: 90%|████████▉ | 3.65GB / 4.06GB  Processing Files (1 / 2) : 90%|████████▉ | 3.66GB / 4.07GB, 181MB/s + New Data Upload : 96%|█████████▌| 3.59GB / 3.76GB, 181MB/s  + + ...b/math/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ath/fft/model.safetensors: 90%|█████████ | 3.68GB / 4.06GB  Processing Files (1 / 2) : 90%|█████████ | 3.69GB / 4.07GB, 181MB/s + New Data Upload : 95%|█████████▍| 3.62GB / 3.82GB, 181MB/s  + + ...b/math/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ath/fft/model.safetensors: 91%|█████████▏| 3.72GB / 4.06GB  Processing Files (1 / 2) : 92%|█████████▏| 3.73GB / 4.07GB, 181MB/s + New Data Upload : 96%|█████████▌| 3.66GB / 3.82GB, 181MB/s  + + ...b/math/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ath/fft/model.safetensors: 92%|█████████▏| 3.75GB / 4.06GB  Processing Files (1 / 2) : 92%|█████████▏| 3.76GB / 4.07GB, 181MB/s + New Data Upload : 95%|█████████▌| 3.70GB / 3.89GB, 181MB/s  + + ...b/math/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ath/fft/model.safetensors: 93%|█████████▎| 3.79GB / 4.06GB  Processing Files (1 / 2) : 93%|█████████▎| 3.80GB / 4.07GB, 182MB/s + New Data Upload : 94%|█████████▍| 3.74GB / 3.96GB, 182MB/s  + + ...b/math/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ath/fft/model.safetensors: 94%|█████████▍| 3.83GB / 4.06GB  Processing Files (1 / 2) : 94%|█████████▍| 3.84GB / 4.07GB, 182MB/s + New Data Upload : 95%|█████████▌| 3.78GB / 3.96GB, 182MB/s  + + ...b/math/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ath/fft/model.safetensors: 95%|█████████▌| 3.88GB / 4.06GB  Processing Files (1 / 2) : 95%|█████████▌| 3.89GB / 4.07GB, 182MB/s + New Data Upload : 95%|█████████▌| 3.82GB / 4.01GB, 182MB/s  + + + + ...ath/fft/training_args.bin: 8%|▊ | 579B / 7.18kB  + + ...b/math/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ath/fft/model.safetensors: 96%|█████████▋| 3.92GB / 4.06GB  + + + + ...ath/fft/training_args.bin: 8%|▊ | 579B / 7.18kB  Processing Files (1 / 3) : 96%|█████████▋| 3.93GB / 4.07GB, 182MB/s + New Data Upload : 96%|█████████▋| 3.86GB / 4.01GB, 182MB/s  + + ...b/math/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ath/fft/model.safetensors: 97%|█████████▋| 3.96GB / 4.06GB  + + + + ...ath/fft/training_args.bin: 13%|█▎ | 909B / 7.18kB  Processing Files (1 / 3) : 97%|█████████▋| 3.97GB / 4.07GB, 183MB/s + New Data Upload : 97%|█████████▋| 3.90GB / 4.01GB, 183MB/s  + + ...b/math/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ath/fft/model.safetensors: 99%|█████████▊| 4.01GB / 4.06GB  + + + + ...ath/fft/training_args.bin: 33%|███▎ | 2.40kB / 7.18kB  Processing Files (1 / 3) : 99%|█████████▊| 4.02GB / 4.07GB, 184MB/s + New Data Upload : 99%|█████████▊| 3.95GB / 4.01GB, 184MB/s  + + ...b/math/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ath/fft/model.safetensors: 99%|█████████▉| 4.04GB / 4.06GB  + + + + ...ath/fft/training_args.bin: 58%|█████▊ | 4.13kB / 7.18kB  Processing Files (1 / 3) : 99%|█████████▉| 4.05GB / 4.07GB, 184MB/s + New Data Upload : 99%|█████████▉| 3.98GB / 4.01GB, 184MB/s  + + ...b/math/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ath/fft/model.safetensors: 100%|█████████▉| 4.05GB / 4.06GB  + + + + ...ath/fft/training_args.bin: 82%|████████▏ | 5.87kB / 7.18kB  Processing Files (1 / 3) : 100%|█████████▉| 4.06GB / 4.07GB, 184MB/s + New Data Upload : 100%|█████████▉| 4.00GB / 4.01GB, 184MB/s  + + ...b/math/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ath/fft/model.safetensors: 100%|█████████▉| 4.06GB / 4.06GB  + + + + ...ath/fft/training_args.bin: 99%|█████████▉| 7.11kB / 7.18kB  Processing Files (1 / 3) : 100%|█████████▉| 4.07GB / 4.07GB, 182MB/s + New Data Upload : 100%|█████████▉| 4.01GB / 4.01GB, 182MB/s  + + ...b/math/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ath/fft/model.safetensors: 100%|█████████▉| 4.06GB / 4.06GB  + + + + ...ath/fft/training_args.bin: 99%|█████████▉| 7.11kB / 7.18kB  + + ...b/math/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ath/fft/model.safetensors: 100%|█████████▉| 4.06GB / 4.06GB  + + + + ...ath/fft/training_args.bin: 99%|█████████▉| 7.11kB / 7.18kB  Processing Files (1 / 3) : 100%|█████████▉| 4.07GB / 4.07GB, 174MB/s + New Data Upload : 100%|█████████▉| 4.01GB / 4.01GB, 174MB/s  + + ...b/math/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ath/fft/model.safetensors: 100%|██████████| 4.06GB / 4.06GB  + + + + ...ath/fft/training_args.bin: 100%|██████████| 7.18kB / 7.18kB  Processing Files (3 / 3) : 100%|██████████| 4.07GB / 4.07GB, 170MB/s + New Data Upload : 100%|██████████| 4.01GB / 4.01GB, 170MB/s  + + ...b/math/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ath/fft/model.safetensors: 100%|██████████| 4.06GB / 4.06GB  + + + + ...ath/fft/training_args.bin: 100%|██████████| 7.18kB / 7.18kB  + + ...b/math/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ath/fft/model.safetensors: 100%|██████████| 4.06GB / 4.06GB  + + + + ...ath/fft/training_args.bin: 100%|██████████| 7.18kB / 7.18kB  + + ...b/math/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ath/fft/model.safetensors: 100%|██████████| 4.06GB / 4.06GB  + + + + ...ath/fft/training_args.bin: 100%|██████████| 7.18kB / 7.18kB  + + ...b/math/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ath/fft/model.safetensors: 100%|██████████| 4.06GB / 4.06GB  + + + + ...ath/fft/training_args.bin: 100%|██████████| 7.18kB / 7.18kB  + + ...b/math/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ath/fft/model.safetensors: 100%|██████████| 4.06GB / 4.06GB  + + + + ...ath/fft/training_args.bin: 100%|██████████| 7.18kB / 7.18kB  + + ...b/math/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ath/fft/model.safetensors: 100%|██████████| 4.06GB / 4.06GB  + + + + ...ath/fft/training_args.bin: 100%|██████████| 7.18kB / 7.18kB  + + ...b/math/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ath/fft/model.safetensors: 100%|██████████| 4.06GB / 4.06GB  + + + + ...ath/fft/training_args.bin: 100%|██████████| 7.18kB / 7.18kB  + + ...b/math/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ath/fft/model.safetensors: 100%|██████████| 4.06GB / 4.06GB  + + + + ...ath/fft/training_args.bin: 100%|██████████| 7.18kB / 7.18kB  Processing Files (3 / 3) : 100%|██████████| 4.07GB / 4.07GB, 143MB/s + New Data Upload : 100%|██████████| 4.01GB / 4.01GB, 143MB/s + ...b/math/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB + ...ath/fft/model.safetensors: 100%|██████████| 4.06GB / 4.06GB + ...ath/fft/training_args.bin: 100%|██████████| 7.18kB / 7.18kB +***** train metrics ***** + epoch = 2.0 + total_flos = 1136301567GF + train_loss = 0.2964 + train_runtime = 1:05:15.33 + train_samples_per_second = 30.649 + train_steps_per_second = 0.24 +Figure saved at: saves/qwen3_1.7b/math/fft/training_loss.png +[WARNING|2026-04-29 16:50:51] llamafactory.extras.ploting:149 >> No metric eval_loss to plot. +[WARNING|2026-04-29 16:50:51] llamafactory.extras.ploting:149 >> No metric eval_accuracy to plot. +[INFO|trainer.py:3797] 2026-04-29 16:50:52,761 >> Saving model checkpoint to saves/qwen3_1.7b/math/fft +[INFO|configuration_utils.py:432] 2026-04-29 16:50:52,769 >> Configuration saved in saves/qwen3_1.7b/math/fft/config.json +[INFO|configuration_utils.py:803] 2026-04-29 16:50:52,775 >> Configuration saved in saves/qwen3_1.7b/math/fft/generation_config.json + Writing model shards: 0%| | 0/1 [00:00> Model weights saved in saves/qwen3_1.7b/math/fft/model.safetensors +[INFO|tokenization_utils_base.py:3224] 2026-04-29 16:50:59,801 >> chat template saved in saves/qwen3_1.7b/math/fft/chat_template.jinja +[INFO|tokenization_utils_base.py:2078] 2026-04-29 16:50:59,805 >> tokenizer config file saved in saves/qwen3_1.7b/math/fft/tokenizer_config.json +[INFO|modelcard.py:266] 2026-04-29 16:51:00,080 >> Dropping the following result as it does not have all the necessary fields: +{'task': {'name': 'Causal Language Modeling', 'type': 'text-generation'}} + Processing Files (0 / 0) : | | 0.00B / 0.00B + New Data Upload : | | 0.00B / 0.00B  + + ...ath/fft/training_args.bin: 100%|██████████| 7.18kB / 7.18kB  + + + ...b/math/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + + ...ath/fft/model.safetensors: 2%|▏ | 72.0MB / 4.06GB  + + ...ath/fft/training_args.bin: 100%|██████████| 7.18kB / 7.18kB  + + + ...b/math/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + + ...ath/fft/model.safetensors: 2%|▏ | 72.0MB / 4.06GB  Processing Files (2 / 3) : 2%|▏ | 83.4MB / 4.07GB, ???B/s + + ...ath/fft/training_args.bin: 100%|██████████| 7.18kB / 7.18kB  + + + ...b/math/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + + ...ath/fft/model.safetensors: 5%|▍ | 184MB / 4.06GB  Processing Files (2 / 3) : 5%|▍ | 195MB / 4.07GB, 561MB/s + + ...ath/fft/training_args.bin: 100%|██████████| 7.18kB / 7.18kB  + + + ...b/math/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + + ...ath/fft/model.safetensors: 8%|▊ | 320MB / 4.06GB  Processing Files (2 / 3) : 8%|▊ | 331MB / 4.07GB, 619MB/s + + ...ath/fft/training_args.bin: 100%|██████████| 7.18kB / 7.18kB  + + + ...b/math/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + + ...ath/fft/model.safetensors: 11%|█ | 448MB / 4.06GB  Processing Files (2 / 3) : 11%|█▏ | 459MB / 4.07GB, 627MB/s + + ...ath/fft/training_args.bin: 100%|██████████| 7.18kB / 7.18kB  + + + ...b/math/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + + ...ath/fft/model.safetensors: 14%|█▍ | 576MB / 4.06GB  Processing Files (2 / 3) : 14%|█▍ | 587MB / 4.07GB, 630MB/s + + ...ath/fft/training_args.bin: 100%|██████████| 7.18kB / 7.18kB  + + + ...b/math/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + + ...ath/fft/model.safetensors: 18%|█▊ | 712MB / 4.06GB  Processing Files (2 / 3) : 18%|█▊ | 723MB / 4.07GB, 640MB/s + + ...ath/fft/training_args.bin: 100%|██████████| 7.18kB / 7.18kB  + + + ...b/math/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + + ...ath/fft/model.safetensors: 21%|██ | 840MB / 4.06GB  Processing Files (2 / 3) : 21%|██ | 851MB / 4.07GB, 640MB/s + + ...ath/fft/training_args.bin: 100%|██████████| 7.18kB / 7.18kB  + + + ...b/math/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + + ...ath/fft/model.safetensors: 24%|██▍ | 976MB / 4.06GB  Processing Files (2 / 3) : 24%|██▍ | 987MB / 4.07GB, 646MB/s + + ...ath/fft/training_args.bin: 100%|██████████| 7.18kB / 7.18kB  + + + ...b/math/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + + ...ath/fft/model.safetensors: 27%|██▋ | 1.11GB / 4.06GB  Processing Files (2 / 3) : 28%|██▊ | 1.12GB / 4.07GB, 650MB/s + + ...ath/fft/training_args.bin: 100%|██████████| 7.18kB / 7.18kB  + + + ...b/math/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + + ...ath/fft/model.safetensors: 30%|███ | 1.24GB / 4.06GB  Processing Files (2 / 3) : 31%|███ | 1.25GB / 4.07GB, 647MB/s + + ...ath/fft/training_args.bin: 100%|██████████| 7.18kB / 7.18kB  + + + ...b/math/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + + ...ath/fft/model.safetensors: 34%|███▍ | 1.37GB / 4.06GB  Processing Files (2 / 3) : 34%|███▍ | 1.38GB / 4.07GB, 650MB/s + + ...ath/fft/training_args.bin: 100%|██████████| 7.18kB / 7.18kB  + + + ...b/math/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + + ...ath/fft/model.safetensors: 37%|███▋ | 1.51GB / 4.06GB  Processing Files (2 / 3) : 37%|███▋ | 1.52GB / 4.07GB, 653MB/s + + ...ath/fft/training_args.bin: 100%|██████████| 7.18kB / 7.18kB  + + + ...b/math/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + + ...ath/fft/model.safetensors: 40%|████ | 1.64GB / 4.06GB  Processing Files (2 / 3) : 41%|████ | 1.66GB / 4.07GB, 655MB/s + + ...ath/fft/training_args.bin: 100%|██████████| 7.18kB / 7.18kB  + + + ...b/math/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + + ...ath/fft/model.safetensors: 44%|████▍ | 1.78GB / 4.06GB  Processing Files (2 / 3) : 44%|████▍ | 1.79GB / 4.07GB, 657MB/s + + ...ath/fft/training_args.bin: 100%|██████████| 7.18kB / 7.18kB  + + + ...b/math/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + + ...ath/fft/model.safetensors: 47%|████▋ | 1.92GB / 4.06GB  Processing Files (2 / 3) : 47%|████▋ | 1.93GB / 4.07GB, 659MB/s + + ...ath/fft/training_args.bin: 100%|██████████| 7.18kB / 7.18kB  + + + ...b/math/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + + ...ath/fft/model.safetensors: 51%|█████ | 2.06GB / 4.06GB  Processing Files (2 / 3) : 51%|█████ | 2.07GB / 4.07GB, 663MB/s + + ...ath/fft/training_args.bin: 100%|██████████| 7.18kB / 7.18kB  + + + ...b/math/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + + ...ath/fft/model.safetensors: 55%|█████▍ | 2.23GB / 4.06GB  Processing Files (2 / 3) : 55%|█████▍ | 2.24GB / 4.07GB, 674MB/s + + ...ath/fft/training_args.bin: 100%|██████████| 7.18kB / 7.18kB  + + + ...b/math/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + + ...ath/fft/model.safetensors: 59%|█████▉ | 2.40GB / 4.06GB  Processing Files (2 / 3) : 59%|█████▉ | 2.41GB / 4.07GB, 683MB/s + + ...ath/fft/training_args.bin: 100%|██████████| 7.18kB / 7.18kB  + + + ...b/math/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + + ...ath/fft/model.safetensors: 63%|██████▎ | 2.56GB / 4.06GB  Processing Files (2 / 3) : 63%|██████▎ | 2.58GB / 4.07GB, 692MB/s + + ...ath/fft/training_args.bin: 100%|██████████| 7.18kB / 7.18kB  + + + ...b/math/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + + ...ath/fft/model.safetensors: 67%|██████▋ | 2.73GB / 4.06GB  Processing Files (2 / 3) : 67%|██████▋ | 2.74GB / 4.07GB, 700MB/s + + ...ath/fft/training_args.bin: 100%|██████████| 7.18kB / 7.18kB  + + + ...b/math/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + + ...ath/fft/model.safetensors: 71%|███████▏ | 2.90GB / 4.06GB  Processing Files (2 / 3) : 71%|███████▏ | 2.91GB / 4.07GB, 707MB/s + + ...ath/fft/training_args.bin: 100%|██████████| 7.18kB / 7.18kB  + + + ...b/math/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + + ...ath/fft/model.safetensors: 75%|███████▌ | 3.07GB / 4.06GB  Processing Files (2 / 3) : 76%|███████▌ | 3.08GB / 4.07GB, 713MB/s + + ...ath/fft/training_args.bin: 100%|██████████| 7.18kB / 7.18kB  + + + ...b/math/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + + ...ath/fft/model.safetensors: 79%|███████▉ | 3.23GB / 4.06GB  Processing Files (2 / 3) : 79%|███████▉ | 3.24GB / 4.07GB, 717MB/s + + ...ath/fft/training_args.bin: 100%|██████████| 7.18kB / 7.18kB  + + + ...b/math/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + + ...ath/fft/model.safetensors: 84%|████████▎ | 3.40GB / 4.06GB  Processing Files (2 / 3) : 84%|████████▎ | 3.41GB / 4.07GB, 723MB/s + + ...ath/fft/training_args.bin: 100%|██████████| 7.18kB / 7.18kB  + + + ...b/math/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + + ...ath/fft/model.safetensors: 88%|████████▊ | 3.56GB / 4.06GB  Processing Files (2 / 3) : 88%|████████▊ | 3.58GB / 4.07GB, 727MB/s + + ...ath/fft/training_args.bin: 100%|██████████| 7.18kB / 7.18kB  + + + ...b/math/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + + ...ath/fft/model.safetensors: 92%|█████████▏| 3.73GB / 4.06GB  Processing Files (2 / 3) : 92%|█████████▏| 3.74GB / 4.07GB, 732MB/s + + ...ath/fft/training_args.bin: 100%|██████████| 7.18kB / 7.18kB  + + + ...b/math/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + + ...ath/fft/model.safetensors: 96%|█████████▌| 3.91GB / 4.06GB  Processing Files (2 / 3) : 96%|█████████▌| 3.92GB / 4.07GB, 738MB/s + + ...ath/fft/training_args.bin: 100%|██████████| 7.18kB / 7.18kB  + + + ...b/math/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + + ...ath/fft/model.safetensors: 100%|█████████▉| 4.06GB / 4.06GB  Processing Files (2 / 3) : 100%|█████████▉| 4.07GB / 4.07GB, 738MB/s + + ...ath/fft/training_args.bin: 100%|██████████| 7.18kB / 7.18kB  + + + ...b/math/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + + ...ath/fft/model.safetensors: 100%|█████████▉| 4.06GB / 4.06GB  Processing Files (2 / 3) : 100%|█████████▉| 4.07GB / 4.07GB, 713MB/s + New Data Upload : 92%|█████████▏| 3.80MB / 4.12MB, 678kB/s  + + ...ath/fft/training_args.bin: 100%|██████████| 7.18kB / 7.18kB  + + + ...b/math/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + + ...ath/fft/model.safetensors: 100%|█████████▉| 4.06GB / 4.06GB  + + ...ath/fft/training_args.bin: 100%|██████████| 7.18kB / 7.18kB  + + + ...b/math/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + + ...ath/fft/model.safetensors: 100%|█████████▉| 4.06GB / 4.06GB  + + ...ath/fft/training_args.bin: 100%|██████████| 7.18kB / 7.18kB  + + + ...b/math/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + + ...ath/fft/model.safetensors: 100%|██████████| 4.06GB / 4.06GB  Processing Files (3 / 3) : 100%|██████████| 4.07GB / 4.07GB, 644MB/s + New Data Upload : 100%|██████████| 4.12MB / 4.12MB, 664kB/s  + + ...ath/fft/training_args.bin: 100%|██████████| 7.18kB / 7.18kB  + + + ...b/math/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + + ...ath/fft/model.safetensors: 100%|██████████| 4.06GB / 4.06GB  + + ...ath/fft/training_args.bin: 100%|██████████| 7.18kB / 7.18kB  + + + ...b/math/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + + ...ath/fft/model.safetensors: 100%|██████████| 4.06GB / 4.06GB  + + ...ath/fft/training_args.bin: 100%|██████████| 7.18kB / 7.18kB  + + + ...b/math/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + + ...ath/fft/model.safetensors: 100%|██████████| 4.06GB / 4.06GB  + + ...ath/fft/training_args.bin: 100%|██████████| 7.18kB / 7.18kB  + + + ...b/math/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + + ...ath/fft/model.safetensors: 100%|██████████| 4.06GB / 4.06GB  + + ...ath/fft/training_args.bin: 100%|██████████| 7.18kB / 7.18kB  + + + ...b/math/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + + ...ath/fft/model.safetensors: 100%|██████████| 4.06GB / 4.06GB  + + ...ath/fft/training_args.bin: 100%|██████████| 7.18kB / 7.18kB  + + + ...b/math/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + + ...ath/fft/model.safetensors: 100%|██████████| 4.06GB / 4.06GB  Processing Files (3 / 3) : 100%|██████████| 4.07GB / 4.07GB, 554MB/s + New Data Upload : 100%|██████████| 4.12MB / 4.12MB, 572kB/s + ...ath/fft/training_args.bin: 100%|██████████| 7.18kB / 7.18kB + ...b/math/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB + ...ath/fft/model.safetensors: 100%|██████████| 4.06GB / 4.06GB diff --git a/tokenizer.json b/tokenizer.json new file mode 100644 index 0000000..c7afbed --- /dev/null +++ b/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:be75606093db2094d7cd20f3c2f385c212750648bd6ea4fb2bf507a6a4c55506 +size 11422650 diff --git a/tokenizer_config.json b/tokenizer_config.json new file mode 100644 index 0000000..145e2c7 --- /dev/null +++ b/tokenizer_config.json @@ -0,0 +1,30 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|im_end|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "padding_side": "right", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/train_results.json b/train_results.json new file mode 100644 index 0000000..8064427 --- /dev/null +++ b/train_results.json @@ -0,0 +1,8 @@ +{ + "epoch": 2.0, + "total_flos": 1.2200945171646382e+18, + "train_loss": 0.2963919332032519, + "train_runtime": 3915.3376, + "train_samples_per_second": 30.649, + "train_steps_per_second": 0.24 +} \ No newline at end of file diff --git a/trainer_log.jsonl b/trainer_log.jsonl new file mode 100644 index 0000000..f036dd9 --- /dev/null +++ b/trainer_log.jsonl @@ -0,0 +1,94 @@ +{"current_steps": 10, "total_steps": 938, "loss": 0.6841894149780273, "lr": 9.574468085106384e-07, "epoch": 0.021333333333333333, "percentage": 1.07, "elapsed_time": "0:00:43", "remaining_time": "1:07:39"} +{"current_steps": 20, "total_steps": 938, "loss": 0.6096980094909668, "lr": 2.021276595744681e-06, "epoch": 0.042666666666666665, "percentage": 2.13, "elapsed_time": "0:01:23", "remaining_time": "1:04:01"} +{"current_steps": 30, "total_steps": 938, "loss": 0.46523351669311525, "lr": 3.0851063829787237e-06, "epoch": 0.064, "percentage": 3.2, "elapsed_time": "0:02:03", "remaining_time": "1:02:05"} +{"current_steps": 40, "total_steps": 938, "loss": 0.38690838813781736, "lr": 4.148936170212766e-06, "epoch": 0.08533333333333333, "percentage": 4.26, "elapsed_time": "0:02:43", "remaining_time": "1:01:20"} +{"current_steps": 50, "total_steps": 938, "loss": 0.3463115692138672, "lr": 5.212765957446809e-06, "epoch": 0.10666666666666667, "percentage": 5.33, "elapsed_time": "0:03:26", "remaining_time": "1:01:14"} +{"current_steps": 60, "total_steps": 938, "loss": 0.3305189609527588, "lr": 6.276595744680851e-06, "epoch": 0.128, "percentage": 6.4, "elapsed_time": "0:04:07", "remaining_time": "1:00:18"} +{"current_steps": 70, "total_steps": 938, "loss": 0.33223772048950195, "lr": 7.340425531914894e-06, "epoch": 0.14933333333333335, "percentage": 7.46, "elapsed_time": "0:04:49", "remaining_time": "0:59:44"} +{"current_steps": 80, "total_steps": 938, "loss": 0.31659588813781736, "lr": 8.404255319148937e-06, "epoch": 0.17066666666666666, "percentage": 8.53, "elapsed_time": "0:05:31", "remaining_time": "0:59:11"} +{"current_steps": 90, "total_steps": 938, "loss": 0.31022181510925295, "lr": 9.46808510638298e-06, "epoch": 0.192, "percentage": 9.59, "elapsed_time": "0:06:11", "remaining_time": "0:58:21"} +{"current_steps": 100, "total_steps": 938, "loss": 0.3155974388122559, "lr": 9.999134070902206e-06, "epoch": 0.21333333333333335, "percentage": 10.66, "elapsed_time": "0:06:52", "remaining_time": "0:57:34"} +{"current_steps": 110, "total_steps": 938, "loss": 0.31209163665771483, "lr": 9.992208437615651e-06, "epoch": 0.23466666666666666, "percentage": 11.73, "elapsed_time": "0:07:33", "remaining_time": "0:56:57"} +{"current_steps": 120, "total_steps": 938, "loss": 0.30946059226989747, "lr": 9.97836676558346e-06, "epoch": 0.256, "percentage": 12.79, "elapsed_time": "0:08:14", "remaining_time": "0:56:12"} +{"current_steps": 130, "total_steps": 938, "loss": 0.3128779888153076, "lr": 9.957628230595527e-06, "epoch": 0.2773333333333333, "percentage": 13.86, "elapsed_time": "0:08:54", "remaining_time": "0:55:20"} +{"current_steps": 140, "total_steps": 938, "loss": 0.3062174320220947, "lr": 9.930021563125204e-06, "epoch": 0.2986666666666667, "percentage": 14.93, "elapsed_time": "0:09:35", "remaining_time": "0:54:39"} +{"current_steps": 150, "total_steps": 938, "loss": 0.30686254501342775, "lr": 9.895585008527075e-06, "epoch": 0.32, "percentage": 15.99, "elapsed_time": "0:10:15", "remaining_time": "0:53:53"} +{"current_steps": 160, "total_steps": 938, "loss": 0.3146007299423218, "lr": 9.854366274053125e-06, "epoch": 0.3413333333333333, "percentage": 17.06, "elapsed_time": "0:10:56", "remaining_time": "0:53:11"} +{"current_steps": 170, "total_steps": 938, "loss": 0.30956292152404785, "lr": 9.806422462760687e-06, "epoch": 0.3626666666666667, "percentage": 18.12, "elapsed_time": "0:11:36", "remaining_time": "0:52:27"} +{"current_steps": 180, "total_steps": 938, "loss": 0.30700514316558836, "lr": 9.751819994403802e-06, "epoch": 0.384, "percentage": 19.19, "elapsed_time": "0:12:19", "remaining_time": "0:51:54"} +{"current_steps": 190, "total_steps": 938, "loss": 0.31289896965026853, "lr": 9.690634513417487e-06, "epoch": 0.4053333333333333, "percentage": 20.26, "elapsed_time": "0:12:59", "remaining_time": "0:51:09"} +{"current_steps": 200, "total_steps": 938, "loss": 0.3092350959777832, "lr": 9.622950784122473e-06, "epoch": 0.4266666666666667, "percentage": 21.32, "elapsed_time": "0:13:40", "remaining_time": "0:50:26"} +{"current_steps": 210, "total_steps": 938, "loss": 0.3032077312469482, "lr": 9.548862573295552e-06, "epoch": 0.448, "percentage": 22.39, "elapsed_time": "0:14:22", "remaining_time": "0:49:50"} +{"current_steps": 220, "total_steps": 938, "loss": 0.30856895446777344, "lr": 9.468472520268207e-06, "epoch": 0.4693333333333333, "percentage": 23.45, "elapsed_time": "0:15:04", "remaining_time": "0:49:13"} +{"current_steps": 230, "total_steps": 938, "loss": 0.3001561641693115, "lr": 9.38189199473352e-06, "epoch": 0.49066666666666664, "percentage": 24.52, "elapsed_time": "0:15:45", "remaining_time": "0:48:31"} +{"current_steps": 240, "total_steps": 938, "loss": 0.2990098476409912, "lr": 9.289240942458322e-06, "epoch": 0.512, "percentage": 25.59, "elapsed_time": "0:16:25", "remaining_time": "0:47:45"} +{"current_steps": 250, "total_steps": 938, "loss": 0.2924530029296875, "lr": 9.190647719114328e-06, "epoch": 0.5333333333333333, "percentage": 26.65, "elapsed_time": "0:17:08", "remaining_time": "0:47:09"} +{"current_steps": 260, "total_steps": 938, "loss": 0.3162691116333008, "lr": 9.086248912458484e-06, "epoch": 0.5546666666666666, "percentage": 27.72, "elapsed_time": "0:17:48", "remaining_time": "0:46:25"} +{"current_steps": 270, "total_steps": 938, "loss": 0.30709133148193357, "lr": 8.976189153108853e-06, "epoch": 0.576, "percentage": 28.78, "elapsed_time": "0:18:28", "remaining_time": "0:45:42"} +{"current_steps": 280, "total_steps": 938, "loss": 0.3032949924468994, "lr": 8.860620914178188e-06, "epoch": 0.5973333333333334, "percentage": 29.85, "elapsed_time": "0:19:08", "remaining_time": "0:44:59"} +{"current_steps": 290, "total_steps": 938, "loss": 0.29636149406433104, "lr": 8.73970430004278e-06, "epoch": 0.6186666666666667, "percentage": 30.92, "elapsed_time": "0:19:49", "remaining_time": "0:44:18"} +{"current_steps": 300, "total_steps": 938, "loss": 0.3070181369781494, "lr": 8.613606824539198e-06, "epoch": 0.64, "percentage": 31.98, "elapsed_time": "0:20:32", "remaining_time": "0:43:41"} +{"current_steps": 310, "total_steps": 938, "loss": 0.3074164867401123, "lr": 8.482503178896227e-06, "epoch": 0.6613333333333333, "percentage": 33.05, "elapsed_time": "0:21:12", "remaining_time": "0:42:57"} +{"current_steps": 320, "total_steps": 938, "loss": 0.2806987762451172, "lr": 8.34657498972347e-06, "epoch": 0.6826666666666666, "percentage": 34.12, "elapsed_time": "0:21:55", "remaining_time": "0:42:20"} +{"current_steps": 330, "total_steps": 938, "loss": 0.28669133186340334, "lr": 8.206010567391918e-06, "epoch": 0.704, "percentage": 35.18, "elapsed_time": "0:22:37", "remaining_time": "0:41:41"} +{"current_steps": 340, "total_steps": 938, "loss": 0.30783905982971194, "lr": 8.061004645155049e-06, "epoch": 0.7253333333333334, "percentage": 36.25, "elapsed_time": "0:23:23", "remaining_time": "0:41:08"} +{"current_steps": 350, "total_steps": 938, "loss": 0.29372529983520507, "lr": 7.91175810937189e-06, "epoch": 0.7466666666666667, "percentage": 37.31, "elapsed_time": "0:24:06", "remaining_time": "0:40:30"} +{"current_steps": 360, "total_steps": 938, "loss": 0.2857055187225342, "lr": 7.758477721205767e-06, "epoch": 0.768, "percentage": 38.38, "elapsed_time": "0:24:50", "remaining_time": "0:39:52"} +{"current_steps": 370, "total_steps": 938, "loss": 0.2911552429199219, "lr": 7.601375830184297e-06, "epoch": 0.7893333333333333, "percentage": 39.45, "elapsed_time": "0:25:32", "remaining_time": "0:39:12"} +{"current_steps": 380, "total_steps": 938, "loss": 0.28858585357666017, "lr": 7.4406700800174545e-06, "epoch": 0.8106666666666666, "percentage": 40.51, "elapsed_time": "0:26:18", "remaining_time": "0:38:37"} +{"current_steps": 390, "total_steps": 938, "loss": 0.2948747158050537, "lr": 7.276583107081243e-06, "epoch": 0.832, "percentage": 41.58, "elapsed_time": "0:26:59", "remaining_time": "0:37:56"} +{"current_steps": 400, "total_steps": 938, "loss": 0.28877460956573486, "lr": 7.109342231984698e-06, "epoch": 0.8533333333333334, "percentage": 42.64, "elapsed_time": "0:27:43", "remaining_time": "0:37:16"} +{"current_steps": 410, "total_steps": 938, "loss": 0.3021462202072144, "lr": 6.939179144647516e-06, "epoch": 0.8746666666666667, "percentage": 43.71, "elapsed_time": "0:28:23", "remaining_time": "0:36:34"} +{"current_steps": 420, "total_steps": 938, "loss": 0.2928939342498779, "lr": 6.766329583324582e-06, "epoch": 0.896, "percentage": 44.78, "elapsed_time": "0:29:06", "remaining_time": "0:35:53"} +{"current_steps": 430, "total_steps": 938, "loss": 0.28879172801971437, "lr": 6.591033008022067e-06, "epoch": 0.9173333333333333, "percentage": 45.84, "elapsed_time": "0:29:46", "remaining_time": "0:35:10"} +{"current_steps": 440, "total_steps": 938, "loss": 0.2961143493652344, "lr": 6.413532268757537e-06, "epoch": 0.9386666666666666, "percentage": 46.91, "elapsed_time": "0:30:28", "remaining_time": "0:34:29"} +{"current_steps": 450, "total_steps": 938, "loss": 0.2954572200775146, "lr": 6.234073269123654e-06, "epoch": 0.96, "percentage": 47.97, "elapsed_time": "0:31:09", "remaining_time": "0:33:47"} +{"current_steps": 460, "total_steps": 938, "loss": 0.2948294162750244, "lr": 6.052904625621556e-06, "epoch": 0.9813333333333333, "percentage": 49.04, "elapsed_time": "0:31:54", "remaining_time": "0:33:09"} +{"current_steps": 470, "total_steps": 938, "loss": 0.2971747875213623, "lr": 5.870277323235871e-06, "epoch": 1.0021333333333333, "percentage": 50.11, "elapsed_time": "0:32:34", "remaining_time": "0:32:26"} +{"current_steps": 480, "total_steps": 938, "loss": 0.26477513313293455, "lr": 5.686444367728494e-06, "epoch": 1.0234666666666667, "percentage": 51.17, "elapsed_time": "0:33:15", "remaining_time": "0:31:44"} +{"current_steps": 490, "total_steps": 938, "loss": 0.2709561824798584, "lr": 5.501660435132871e-06, "epoch": 1.0448, "percentage": 52.24, "elapsed_time": "0:33:57", "remaining_time": "0:31:02"} +{"current_steps": 500, "total_steps": 938, "loss": 0.27056255340576174, "lr": 5.316181518934319e-06, "epoch": 1.0661333333333334, "percentage": 53.3, "elapsed_time": "0:34:38", "remaining_time": "0:30:20"} +{"current_steps": 510, "total_steps": 938, "loss": 0.27835679054260254, "lr": 5.130264575425225e-06, "epoch": 1.0874666666666666, "percentage": 54.37, "elapsed_time": "0:35:30", "remaining_time": "0:29:48"} +{"current_steps": 520, "total_steps": 938, "loss": 0.26178646087646484, "lr": 4.944167167726367e-06, "epoch": 1.1088, "percentage": 55.44, "elapsed_time": "0:36:13", "remaining_time": "0:29:07"} +{"current_steps": 530, "total_steps": 938, "loss": 0.2711505889892578, "lr": 4.758147108967585e-06, "epoch": 1.1301333333333332, "percentage": 56.5, "elapsed_time": "0:36:54", "remaining_time": "0:28:24"} +{"current_steps": 540, "total_steps": 938, "loss": 0.262709903717041, "lr": 4.572462105122078e-06, "epoch": 1.1514666666666666, "percentage": 57.57, "elapsed_time": "0:37:37", "remaining_time": "0:27:43"} +{"current_steps": 550, "total_steps": 938, "loss": 0.2738328218460083, "lr": 4.3873693979891705e-06, "epoch": 1.1728, "percentage": 58.64, "elapsed_time": "0:38:18", "remaining_time": "0:27:01"} +{"current_steps": 560, "total_steps": 938, "loss": 0.26540517807006836, "lr": 4.203125408820106e-06, "epoch": 1.1941333333333333, "percentage": 59.7, "elapsed_time": "0:38:58", "remaining_time": "0:26:18"} +{"current_steps": 570, "total_steps": 938, "loss": 0.26726350784301756, "lr": 4.019985383080632e-06, "epoch": 1.2154666666666667, "percentage": 60.77, "elapsed_time": "0:39:40", "remaining_time": "0:25:37"} +{"current_steps": 580, "total_steps": 938, "loss": 0.2629551887512207, "lr": 3.838203036842446e-06, "epoch": 1.2368000000000001, "percentage": 61.83, "elapsed_time": "0:40:20", "remaining_time": "0:24:53"} +{"current_steps": 590, "total_steps": 938, "loss": 0.2717682600021362, "lr": 3.65803020529343e-06, "epoch": 1.2581333333333333, "percentage": 62.9, "elapsed_time": "0:41:04", "remaining_time": "0:24:13"} +{"current_steps": 600, "total_steps": 938, "loss": 0.2709221601486206, "lr": 3.4797164938536113e-06, "epoch": 1.2794666666666665, "percentage": 63.97, "elapsed_time": "0:41:46", "remaining_time": "0:23:32"} +{"current_steps": 610, "total_steps": 938, "loss": 0.26828670501708984, "lr": 3.3035089323801323e-06, "epoch": 1.3008, "percentage": 65.03, "elapsed_time": "0:42:29", "remaining_time": "0:22:50"} +{"current_steps": 620, "total_steps": 938, "loss": 0.27147223949432375, "lr": 3.1296516329403626e-06, "epoch": 1.3221333333333334, "percentage": 66.1, "elapsed_time": "0:43:10", "remaining_time": "0:22:08"} +{"current_steps": 630, "total_steps": 938, "loss": 0.272589111328125, "lr": 2.9583854516271815e-06, "epoch": 1.3434666666666666, "percentage": 67.16, "elapsed_time": "0:43:51", "remaining_time": "0:21:26"} +{"current_steps": 640, "total_steps": 938, "loss": 0.2664657115936279, "lr": 2.7899476548850046e-06, "epoch": 1.3648, "percentage": 68.23, "elapsed_time": "0:44:32", "remaining_time": "0:20:44"} +{"current_steps": 650, "total_steps": 938, "loss": 0.2647489786148071, "lr": 2.624571590808781e-06, "epoch": 1.3861333333333334, "percentage": 69.3, "elapsed_time": "0:45:12", "remaining_time": "0:20:01"} +{"current_steps": 660, "total_steps": 938, "loss": 0.2691283941268921, "lr": 2.4624863658713383e-06, "epoch": 1.4074666666666666, "percentage": 70.36, "elapsed_time": "0:45:53", "remaining_time": "0:19:19"} +{"current_steps": 670, "total_steps": 938, "loss": 0.2635840892791748, "lr": 2.3039165275269214e-06, "epoch": 1.4288, "percentage": 71.43, "elapsed_time": "0:46:34", "remaining_time": "0:18:37"} +{"current_steps": 680, "total_steps": 938, "loss": 0.2607363224029541, "lr": 2.1490817531306778e-06, "epoch": 1.4501333333333333, "percentage": 72.49, "elapsed_time": "0:47:15", "remaining_time": "0:17:55"} +{"current_steps": 690, "total_steps": 938, "loss": 0.27231812477111816, "lr": 1.99819654560496e-06, "epoch": 1.4714666666666667, "percentage": 73.56, "elapsed_time": "0:47:54", "remaining_time": "0:17:13"} +{"current_steps": 700, "total_steps": 938, "loss": 0.2777836561203003, "lr": 1.851469936274174e-06, "epoch": 1.4928, "percentage": 74.63, "elapsed_time": "0:48:35", "remaining_time": "0:16:31"} +{"current_steps": 710, "total_steps": 938, "loss": 0.2632488965988159, "lr": 1.7091051952797404e-06, "epoch": 1.5141333333333333, "percentage": 75.69, "elapsed_time": "0:49:18", "remaining_time": "0:15:50"} +{"current_steps": 720, "total_steps": 938, "loss": 0.2699136257171631, "lr": 1.571299549976456e-06, "epoch": 1.5354666666666668, "percentage": 76.76, "elapsed_time": "0:49:58", "remaining_time": "0:15:07"} +{"current_steps": 730, "total_steps": 938, "loss": 0.25556948184967043, "lr": 1.4382439117002938e-06, "epoch": 1.5568, "percentage": 77.83, "elapsed_time": "0:50:41", "remaining_time": "0:14:26"} +{"current_steps": 740, "total_steps": 938, "loss": 0.26279048919677733, "lr": 1.3101226112862232e-06, "epoch": 1.5781333333333334, "percentage": 78.89, "elapsed_time": "0:51:22", "remaining_time": "0:13:44"} +{"current_steps": 750, "total_steps": 938, "loss": 0.2726590633392334, "lr": 1.1871131437024291e-06, "epoch": 1.5994666666666668, "percentage": 79.96, "elapsed_time": "0:52:02", "remaining_time": "0:13:02"} +{"current_steps": 760, "total_steps": 938, "loss": 0.2752861499786377, "lr": 1.0693859221547115e-06, "epoch": 1.6208, "percentage": 81.02, "elapsed_time": "0:52:45", "remaining_time": "0:12:21"} +{"current_steps": 770, "total_steps": 938, "loss": 0.2579826354980469, "lr": 9.571040420017324e-07, "epoch": 1.6421333333333332, "percentage": 82.09, "elapsed_time": "0:53:28", "remaining_time": "0:11:39"} +{"current_steps": 780, "total_steps": 938, "loss": 0.26852080821990965, "lr": 8.504230548081499e-07, "epoch": 1.6634666666666666, "percentage": 83.16, "elapsed_time": "0:54:10", "remaining_time": "0:10:58"} +{"current_steps": 790, "total_steps": 938, "loss": 0.26603677272796633, "lr": 7.4949075284868e-07, "epoch": 1.6848, "percentage": 84.22, "elapsed_time": "0:54:52", "remaining_time": "0:10:16"} +{"current_steps": 800, "total_steps": 938, "loss": 0.25719456672668456, "lr": 6.544469643616192e-07, "epoch": 1.7061333333333333, "percentage": 85.29, "elapsed_time": "0:55:34", "remaining_time": "0:09:35"} +{"current_steps": 810, "total_steps": 938, "loss": 0.2663470506668091, "lr": 5.654233598354752e-07, "epoch": 1.7274666666666667, "percentage": 86.35, "elapsed_time": "0:56:15", "remaining_time": "0:08:53"} +{"current_steps": 820, "total_steps": 938, "loss": 0.26163082122802733, "lr": 4.825432695970673e-07, "epoch": 1.7488000000000001, "percentage": 87.42, "elapsed_time": "0:57:00", "remaining_time": "0:08:12"} +{"current_steps": 830, "total_steps": 938, "loss": 0.26932682991027834, "lr": 4.0592151295382464e-07, "epoch": 1.7701333333333333, "percentage": 88.49, "elapsed_time": "0:57:40", "remaining_time": "0:07:30"} +{"current_steps": 840, "total_steps": 938, "loss": 0.27272663116455076, "lr": 3.356642391269405e-07, "epoch": 1.7914666666666665, "percentage": 89.55, "elapsed_time": "0:58:21", "remaining_time": "0:06:48"} +{"current_steps": 850, "total_steps": 938, "loss": 0.2759810447692871, "lr": 2.7186878019580197e-07, "epoch": 1.8128, "percentage": 90.62, "elapsed_time": "0:59:02", "remaining_time": "0:06:06"} +{"current_steps": 860, "total_steps": 938, "loss": 0.2736822128295898, "lr": 2.1462351625736677e-07, "epoch": 1.8341333333333334, "percentage": 91.68, "elapsed_time": "0:59:43", "remaining_time": "0:05:24"} +{"current_steps": 870, "total_steps": 938, "loss": 0.2506378173828125, "lr": 1.6400775298734017e-07, "epoch": 1.8554666666666666, "percentage": 92.75, "elapsed_time": "1:00:28", "remaining_time": "0:04:43"} +{"current_steps": 880, "total_steps": 938, "loss": 0.2735074520111084, "lr": 1.2009161177273742e-07, "epoch": 1.8768, "percentage": 93.82, "elapsed_time": "1:01:08", "remaining_time": "0:04:01"} +{"current_steps": 890, "total_steps": 938, "loss": 0.26666769981384275, "lr": 8.293593256805842e-08, "epoch": 1.8981333333333335, "percentage": 94.88, "elapsed_time": "1:01:49", "remaining_time": "0:03:20"} +{"current_steps": 900, "total_steps": 938, "loss": 0.2759205102920532, "lr": 5.259218960964874e-08, "epoch": 1.9194666666666667, "percentage": 95.95, "elapsed_time": "1:02:31", "remaining_time": "0:02:38"} +{"current_steps": 910, "total_steps": 938, "loss": 0.27174739837646483, "lr": 2.9102420105009966e-08, "epoch": 1.9407999999999999, "percentage": 97.01, "elapsed_time": "1:03:12", "remaining_time": "0:01:56"} +{"current_steps": 920, "total_steps": 938, "loss": 0.26730976104736326, "lr": 1.249916599585954e-08, "epoch": 1.9621333333333333, "percentage": 98.08, "elapsed_time": "1:03:50", "remaining_time": "0:01:14"} +{"current_steps": 930, "total_steps": 938, "loss": 0.2772205352783203, "lr": 2.80542887561297e-09, "epoch": 1.9834666666666667, "percentage": 99.15, "elapsed_time": "1:04:31", "remaining_time": "0:00:33"} +{"current_steps": 938, "total_steps": 938, "epoch": 2.0, "percentage": 100.0, "elapsed_time": "1:05:13", "remaining_time": "0:00:00"} diff --git a/trainer_state.json b/trainer_state.json new file mode 100644 index 0000000..b4d4be0 --- /dev/null +++ b/trainer_state.json @@ -0,0 +1,694 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 2.0, + "eval_steps": 500, + "global_step": 938, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.021333333333333333, + "grad_norm": 11.143438339233398, + "learning_rate": 9.574468085106384e-07, + "loss": 0.6841894149780273, + "step": 10 + }, + { + "epoch": 0.042666666666666665, + "grad_norm": 4.5582380294799805, + "learning_rate": 2.021276595744681e-06, + "loss": 0.6096980094909668, + "step": 20 + }, + { + "epoch": 0.064, + "grad_norm": 1.4977697134017944, + "learning_rate": 3.0851063829787237e-06, + "loss": 0.46523351669311525, + "step": 30 + }, + { + "epoch": 0.08533333333333333, + "grad_norm": 0.7574602961540222, + "learning_rate": 4.148936170212766e-06, + "loss": 0.38690838813781736, + "step": 40 + }, + { + "epoch": 0.10666666666666667, + "grad_norm": 0.6511842012405396, + "learning_rate": 5.212765957446809e-06, + "loss": 0.3463115692138672, + "step": 50 + }, + { + "epoch": 0.128, + "grad_norm": 0.5265598297119141, + "learning_rate": 6.276595744680851e-06, + "loss": 0.3305189609527588, + "step": 60 + }, + { + "epoch": 0.14933333333333335, + "grad_norm": 0.5384871363639832, + "learning_rate": 7.340425531914894e-06, + "loss": 0.33223772048950195, + "step": 70 + }, + { + "epoch": 0.17066666666666666, + "grad_norm": 0.5109423995018005, + "learning_rate": 8.404255319148937e-06, + "loss": 0.31659588813781736, + "step": 80 + }, + { + "epoch": 0.192, + "grad_norm": 0.47720399498939514, + "learning_rate": 9.46808510638298e-06, + "loss": 0.31022181510925295, + "step": 90 + }, + { + "epoch": 0.21333333333333335, + "grad_norm": 0.49756094813346863, + "learning_rate": 9.999134070902206e-06, + "loss": 0.3155974388122559, + "step": 100 + }, + { + "epoch": 0.23466666666666666, + "grad_norm": 0.4745752215385437, + "learning_rate": 9.992208437615651e-06, + "loss": 0.31209163665771483, + "step": 110 + }, + { + "epoch": 0.256, + "grad_norm": 0.4676297903060913, + "learning_rate": 9.97836676558346e-06, + "loss": 0.30946059226989747, + "step": 120 + }, + { + "epoch": 0.2773333333333333, + "grad_norm": 0.4762783944606781, + "learning_rate": 9.957628230595527e-06, + "loss": 0.3128779888153076, + "step": 130 + }, + { + "epoch": 0.2986666666666667, + "grad_norm": 0.4929763078689575, + "learning_rate": 9.930021563125204e-06, + "loss": 0.3062174320220947, + "step": 140 + }, + { + "epoch": 0.32, + "grad_norm": 0.4954136312007904, + "learning_rate": 9.895585008527075e-06, + "loss": 0.30686254501342775, + "step": 150 + }, + { + "epoch": 0.3413333333333333, + "grad_norm": 0.48502442240715027, + "learning_rate": 9.854366274053125e-06, + "loss": 0.3146007299423218, + "step": 160 + }, + { + "epoch": 0.3626666666666667, + "grad_norm": 0.4601064622402191, + "learning_rate": 9.806422462760687e-06, + "loss": 0.30956292152404785, + "step": 170 + }, + { + "epoch": 0.384, + "grad_norm": 0.4399697780609131, + "learning_rate": 9.751819994403802e-06, + "loss": 0.30700514316558836, + "step": 180 + }, + { + "epoch": 0.4053333333333333, + "grad_norm": 0.5086579322814941, + "learning_rate": 9.690634513417487e-06, + "loss": 0.31289896965026853, + "step": 190 + }, + { + "epoch": 0.4266666666666667, + "grad_norm": 0.4878501892089844, + "learning_rate": 9.622950784122473e-06, + "loss": 0.3092350959777832, + "step": 200 + }, + { + "epoch": 0.448, + "grad_norm": 0.47980615496635437, + "learning_rate": 9.548862573295552e-06, + "loss": 0.3032077312469482, + "step": 210 + }, + { + "epoch": 0.4693333333333333, + "grad_norm": 0.4887213110923767, + "learning_rate": 9.468472520268207e-06, + "loss": 0.30856895446777344, + "step": 220 + }, + { + "epoch": 0.49066666666666664, + "grad_norm": 0.48731157183647156, + "learning_rate": 9.38189199473352e-06, + "loss": 0.3001561641693115, + "step": 230 + }, + { + "epoch": 0.512, + "grad_norm": 0.5213120579719543, + "learning_rate": 9.289240942458322e-06, + "loss": 0.2990098476409912, + "step": 240 + }, + { + "epoch": 0.5333333333333333, + "grad_norm": 0.46841681003570557, + "learning_rate": 9.190647719114328e-06, + "loss": 0.2924530029296875, + "step": 250 + }, + { + "epoch": 0.5546666666666666, + "grad_norm": 0.45176810026168823, + "learning_rate": 9.086248912458484e-06, + "loss": 0.3162691116333008, + "step": 260 + }, + { + "epoch": 0.576, + "grad_norm": 0.4480721056461334, + "learning_rate": 8.976189153108853e-06, + "loss": 0.30709133148193357, + "step": 270 + }, + { + "epoch": 0.5973333333333334, + "grad_norm": 0.4673004150390625, + "learning_rate": 8.860620914178188e-06, + "loss": 0.3032949924468994, + "step": 280 + }, + { + "epoch": 0.6186666666666667, + "grad_norm": 0.4601098299026489, + "learning_rate": 8.73970430004278e-06, + "loss": 0.29636149406433104, + "step": 290 + }, + { + "epoch": 0.64, + "grad_norm": 0.4462672173976898, + "learning_rate": 8.613606824539198e-06, + "loss": 0.3070181369781494, + "step": 300 + }, + { + "epoch": 0.6613333333333333, + "grad_norm": 0.48093080520629883, + "learning_rate": 8.482503178896227e-06, + "loss": 0.3074164867401123, + "step": 310 + }, + { + "epoch": 0.6826666666666666, + "grad_norm": 0.5080389380455017, + "learning_rate": 8.34657498972347e-06, + "loss": 0.2806987762451172, + "step": 320 + }, + { + "epoch": 0.704, + "grad_norm": 0.46668171882629395, + "learning_rate": 8.206010567391918e-06, + "loss": 0.28669133186340334, + "step": 330 + }, + { + "epoch": 0.7253333333333334, + "grad_norm": 0.4524853527545929, + "learning_rate": 8.061004645155049e-06, + "loss": 0.30783905982971194, + "step": 340 + }, + { + "epoch": 0.7466666666666667, + "grad_norm": 0.5166858434677124, + "learning_rate": 7.91175810937189e-06, + "loss": 0.29372529983520507, + "step": 350 + }, + { + "epoch": 0.768, + "grad_norm": 0.46780723333358765, + "learning_rate": 7.758477721205767e-06, + "loss": 0.2857055187225342, + "step": 360 + }, + { + "epoch": 0.7893333333333333, + "grad_norm": 0.47541725635528564, + "learning_rate": 7.601375830184297e-06, + "loss": 0.2911552429199219, + "step": 370 + }, + { + "epoch": 0.8106666666666666, + "grad_norm": 0.44671547412872314, + "learning_rate": 7.4406700800174545e-06, + "loss": 0.28858585357666017, + "step": 380 + }, + { + "epoch": 0.832, + "grad_norm": 0.44009825587272644, + "learning_rate": 7.276583107081243e-06, + "loss": 0.2948747158050537, + "step": 390 + }, + { + "epoch": 0.8533333333333334, + "grad_norm": 0.4630362093448639, + "learning_rate": 7.109342231984698e-06, + "loss": 0.28877460956573486, + "step": 400 + }, + { + "epoch": 0.8746666666666667, + "grad_norm": 0.506441593170166, + "learning_rate": 6.939179144647516e-06, + "loss": 0.3021462202072144, + "step": 410 + }, + { + "epoch": 0.896, + "grad_norm": 0.4811643660068512, + "learning_rate": 6.766329583324582e-06, + "loss": 0.2928939342498779, + "step": 420 + }, + { + "epoch": 0.9173333333333333, + "grad_norm": 0.4531882107257843, + "learning_rate": 6.591033008022067e-06, + "loss": 0.28879172801971437, + "step": 430 + }, + { + "epoch": 0.9386666666666666, + "grad_norm": 0.4446716904640198, + "learning_rate": 6.413532268757537e-06, + "loss": 0.2961143493652344, + "step": 440 + }, + { + "epoch": 0.96, + "grad_norm": 0.41976165771484375, + "learning_rate": 6.234073269123654e-06, + "loss": 0.2954572200775146, + "step": 450 + }, + { + "epoch": 0.9813333333333333, + "grad_norm": 0.4504052698612213, + "learning_rate": 6.052904625621556e-06, + "loss": 0.2948294162750244, + "step": 460 + }, + { + "epoch": 1.0021333333333333, + "grad_norm": 0.5018347501754761, + "learning_rate": 5.870277323235871e-06, + "loss": 0.2971747875213623, + "step": 470 + }, + { + "epoch": 1.0234666666666667, + "grad_norm": 0.48416292667388916, + "learning_rate": 5.686444367728494e-06, + "loss": 0.26477513313293455, + "step": 480 + }, + { + "epoch": 1.0448, + "grad_norm": 0.4401777386665344, + "learning_rate": 5.501660435132871e-06, + "loss": 0.2709561824798584, + "step": 490 + }, + { + "epoch": 1.0661333333333334, + "grad_norm": 0.4749163091182709, + "learning_rate": 5.316181518934319e-06, + "loss": 0.27056255340576174, + "step": 500 + }, + { + "epoch": 1.0874666666666666, + "grad_norm": 0.4591522216796875, + "learning_rate": 5.130264575425225e-06, + "loss": 0.27835679054260254, + "step": 510 + }, + { + "epoch": 1.1088, + "grad_norm": 0.4608944356441498, + "learning_rate": 4.944167167726367e-06, + "loss": 0.26178646087646484, + "step": 520 + }, + { + "epoch": 1.1301333333333332, + "grad_norm": 0.48433825373649597, + "learning_rate": 4.758147108967585e-06, + "loss": 0.2711505889892578, + "step": 530 + }, + { + "epoch": 1.1514666666666666, + "grad_norm": 0.5324186682701111, + "learning_rate": 4.572462105122078e-06, + "loss": 0.262709903717041, + "step": 540 + }, + { + "epoch": 1.1728, + "grad_norm": 0.4596474766731262, + "learning_rate": 4.3873693979891705e-06, + "loss": 0.2738328218460083, + "step": 550 + }, + { + "epoch": 1.1941333333333333, + "grad_norm": 0.48839670419692993, + "learning_rate": 4.203125408820106e-06, + "loss": 0.26540517807006836, + "step": 560 + }, + { + "epoch": 1.2154666666666667, + "grad_norm": 0.4470955729484558, + "learning_rate": 4.019985383080632e-06, + "loss": 0.26726350784301756, + "step": 570 + }, + { + "epoch": 1.2368000000000001, + "grad_norm": 0.44690802693367004, + "learning_rate": 3.838203036842446e-06, + "loss": 0.2629551887512207, + "step": 580 + }, + { + "epoch": 1.2581333333333333, + "grad_norm": 0.4668348431587219, + "learning_rate": 3.65803020529343e-06, + "loss": 0.2717682600021362, + "step": 590 + }, + { + "epoch": 1.2794666666666665, + "grad_norm": 0.440378338098526, + "learning_rate": 3.4797164938536113e-06, + "loss": 0.2709221601486206, + "step": 600 + }, + { + "epoch": 1.3008, + "grad_norm": 0.508844792842865, + "learning_rate": 3.3035089323801323e-06, + "loss": 0.26828670501708984, + "step": 610 + }, + { + "epoch": 1.3221333333333334, + "grad_norm": 0.49634313583374023, + "learning_rate": 3.1296516329403626e-06, + "loss": 0.27147223949432375, + "step": 620 + }, + { + "epoch": 1.3434666666666666, + "grad_norm": 0.4920685887336731, + "learning_rate": 2.9583854516271815e-06, + "loss": 0.272589111328125, + "step": 630 + }, + { + "epoch": 1.3648, + "grad_norm": 0.4857788681983948, + "learning_rate": 2.7899476548850046e-06, + "loss": 0.2664657115936279, + "step": 640 + }, + { + "epoch": 1.3861333333333334, + "grad_norm": 0.45807984471321106, + "learning_rate": 2.624571590808781e-06, + "loss": 0.2647489786148071, + "step": 650 + }, + { + "epoch": 1.4074666666666666, + "grad_norm": 0.4540899097919464, + "learning_rate": 2.4624863658713383e-06, + "loss": 0.2691283941268921, + "step": 660 + }, + { + "epoch": 1.4288, + "grad_norm": 0.5011265873908997, + "learning_rate": 2.3039165275269214e-06, + "loss": 0.2635840892791748, + "step": 670 + }, + { + "epoch": 1.4501333333333333, + "grad_norm": 0.441933810710907, + "learning_rate": 2.1490817531306778e-06, + "loss": 0.2607363224029541, + "step": 680 + }, + { + "epoch": 1.4714666666666667, + "grad_norm": 0.44827017188072205, + "learning_rate": 1.99819654560496e-06, + "loss": 0.27231812477111816, + "step": 690 + }, + { + "epoch": 1.4928, + "grad_norm": 0.4514150619506836, + "learning_rate": 1.851469936274174e-06, + "loss": 0.2777836561203003, + "step": 700 + }, + { + "epoch": 1.5141333333333333, + "grad_norm": 0.47360625863075256, + "learning_rate": 1.7091051952797404e-06, + "loss": 0.2632488965988159, + "step": 710 + }, + { + "epoch": 1.5354666666666668, + "grad_norm": 0.4452371895313263, + "learning_rate": 1.571299549976456e-06, + "loss": 0.2699136257171631, + "step": 720 + }, + { + "epoch": 1.5568, + "grad_norm": 0.45347362756729126, + "learning_rate": 1.4382439117002938e-06, + "loss": 0.25556948184967043, + "step": 730 + }, + { + "epoch": 1.5781333333333334, + "grad_norm": 0.45227858424186707, + "learning_rate": 1.3101226112862232e-06, + "loss": 0.26279048919677733, + "step": 740 + }, + { + "epoch": 1.5994666666666668, + "grad_norm": 0.4334966838359833, + "learning_rate": 1.1871131437024291e-06, + "loss": 0.2726590633392334, + "step": 750 + }, + { + "epoch": 1.6208, + "grad_norm": 0.4579485058784485, + "learning_rate": 1.0693859221547115e-06, + "loss": 0.2752861499786377, + "step": 760 + }, + { + "epoch": 1.6421333333333332, + "grad_norm": 0.48792654275894165, + "learning_rate": 9.571040420017324e-07, + "loss": 0.2579826354980469, + "step": 770 + }, + { + "epoch": 1.6634666666666666, + "grad_norm": 0.4276820421218872, + "learning_rate": 8.504230548081499e-07, + "loss": 0.26852080821990965, + "step": 780 + }, + { + "epoch": 1.6848, + "grad_norm": 0.4804341793060303, + "learning_rate": 7.4949075284868e-07, + "loss": 0.26603677272796633, + "step": 790 + }, + { + "epoch": 1.7061333333333333, + "grad_norm": 0.4831051230430603, + "learning_rate": 6.544469643616192e-07, + "loss": 0.25719456672668456, + "step": 800 + }, + { + "epoch": 1.7274666666666667, + "grad_norm": 0.508553683757782, + "learning_rate": 5.654233598354752e-07, + "loss": 0.2663470506668091, + "step": 810 + }, + { + "epoch": 1.7488000000000001, + "grad_norm": 0.4779980480670929, + "learning_rate": 4.825432695970673e-07, + "loss": 0.26163082122802733, + "step": 820 + }, + { + "epoch": 1.7701333333333333, + "grad_norm": 0.41447553038597107, + "learning_rate": 4.0592151295382464e-07, + "loss": 0.26932682991027834, + "step": 830 + }, + { + "epoch": 1.7914666666666665, + "grad_norm": 0.43118005990982056, + "learning_rate": 3.356642391269405e-07, + "loss": 0.27272663116455076, + "step": 840 + }, + { + "epoch": 1.8128, + "grad_norm": 0.4650283753871918, + "learning_rate": 2.7186878019580197e-07, + "loss": 0.2759810447692871, + "step": 850 + }, + { + "epoch": 1.8341333333333334, + "grad_norm": 0.45311886072158813, + "learning_rate": 2.1462351625736677e-07, + "loss": 0.2736822128295898, + "step": 860 + }, + { + "epoch": 1.8554666666666666, + "grad_norm": 0.43716010451316833, + "learning_rate": 1.6400775298734017e-07, + "loss": 0.2506378173828125, + "step": 870 + }, + { + "epoch": 1.8768, + "grad_norm": 0.46398675441741943, + "learning_rate": 1.2009161177273742e-07, + "loss": 0.2735074520111084, + "step": 880 + }, + { + "epoch": 1.8981333333333335, + "grad_norm": 0.4555061161518097, + "learning_rate": 8.293593256805842e-08, + "loss": 0.26666769981384275, + "step": 890 + }, + { + "epoch": 1.9194666666666667, + "grad_norm": 0.4673771560192108, + "learning_rate": 5.259218960964874e-08, + "loss": 0.2759205102920532, + "step": 900 + }, + { + "epoch": 1.9407999999999999, + "grad_norm": 0.46401599049568176, + "learning_rate": 2.9102420105009966e-08, + "loss": 0.27174739837646483, + "step": 910 + }, + { + "epoch": 1.9621333333333333, + "grad_norm": 0.45422956347465515, + "learning_rate": 1.249916599585954e-08, + "loss": 0.26730976104736326, + "step": 920 + }, + { + "epoch": 1.9834666666666667, + "grad_norm": 0.4789884686470032, + "learning_rate": 2.80542887561297e-09, + "loss": 0.2772205352783203, + "step": 930 + }, + { + "epoch": 2.0, + "step": 938, + "total_flos": 1.2200945171646382e+18, + "train_loss": 0.2963919332032519, + "train_runtime": 3915.3376, + "train_samples_per_second": 30.649, + "train_steps_per_second": 0.24 + } + ], + "logging_steps": 10, + "max_steps": 938, + "num_input_tokens_seen": 0, + "num_train_epochs": 2, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": true + }, + "attributes": {} + } + }, + "total_flos": 1.2200945171646382e+18, + "train_batch_size": 2, + "trial_name": null, + "trial_params": null +} diff --git a/training_args.bin b/training_args.bin new file mode 100644 index 0000000..4bcd27f --- /dev/null +++ b/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cceab43f5c9cf083825a08d73354a3abf6cb2e33b74d60669f851f401d270ebd +size 7185 diff --git a/training_loss.png b/training_loss.png new file mode 100644 index 0000000000000000000000000000000000000000..3eaa93fda2e473ea69b185bd9a89b85f0e0cc298 GIT binary patch literal 35377 zcmdqJbyQW`*Ehb;p+j1_6-1B{l}=HRP{9CcK|ulO&I3r7h=?c%q9`3A(r}PeBvcTP z4yC)L-nsGlKF>SGFUI@#dx!UW$2ohiz4lyd&iR?2&s-<`>SeVfh(W@07yHXW%))ZtqOGiD?lzQ$4# zaHF^5ZX?IxYjUywTW)^bp>gR_-F)e=pGilGjH}?=%PGGy)<5W3%`6;=;Zlrg6{(~& z$YTuT9(iG3v+QL1<@*6GGL)C0xBfPfc;KN6cXxOEl2sNLc zs{4{4RAfJ_KuAPXvSW0xzgb)4v+WecdX{{7d6^^IqBBi-YwgFC7aRxh-y*kp10%P- zyBspZeK+%s!gKN~vad}}W&Zu^CAa!j0_o(dCf6RW{~Uk9ai;6!sax*}{r6W>2V-v) zTOP5tu~{B)?%)v;YW4LQ{S;ib|94g#mgI7e@_sDlkrz>DJtnddo1oFXFe#AmWz|nU zSjBOu8?!;Ed-wAqSBXvRZes)?~ly;T#A1#|xJ^!%0dH2p@uD57jp{LKu$>~u_ zO0@g1?`BoU=mV?T(D(08BW(vYC2q-?$9jph2FIq~r8d2eRUk*ywY6Q{mnVMz#vp{s z_}%T*iITZF?{&eWBIOfw$x@E#eKw-7 zDSUS}f(`O6T3YgdwC($zE*7&$f`B|UQF zf`fw?`rzHavjqz~uoLeeKXHNp;q+b8Aq{kBU#NBN_J)NW@LsHD7BfS-tj|JVf*mbc zrg#Eb>bI{2hxA$be4?%WU_4e)g@$lqqMT!)MQ!1rjtUIpvtSV8l=x z=Ue4Y8EUhs0udhj+mmt6o{_@4M2b|9hEj2?Y{~uX>QWL+7;BRpc2O)yPsh@Q6GA+s zx$w<*_oerOvaM}yv&}+p5kaDllc2`N7J9Ftx%s?EqWhHegT8X-6L4LCp?!5pO#}7)GAk z5~TH!0a4>-1vKE#^SFiIQERTT102<6cIhDXmsr{4n^mxc(6^qzh2p^O-*BW7Hhd+fwv~& zuQ;rC>!Hr`sien*0?~hehiswtzz_9wkC6Zpv(rh1{s(^G!vv)RP87(U^k}=gOUjAt7U;{@WB2PSQi(i}G42GDj5; zb|spk*+OUYE2(5ssHo-FXzXgXEwxe=s@pIN^I7ga`Nq8;AF$a~O!FV-=jVl&U#wcC zf|M0eA4+}x^*6Jx&%~$tOTh+X6q`(`tt-D&fbJ|&(0QPSMcc_OfS3J%Bb~Hu5WJuJeEw)$dTw` zENAq)mtN%HFkFb**g_y4Y)vX2 zOfO66FzDR6MfRdCWV|_ce7Tt`^t#~suR8*XFgJF@fD-3c$$`3?NZAM$U1&7!z4V$n z*=u_))&?tOXf0a3rrUDorKwG^W!tAT<)EdpNKtd}9@N2UIXPoChrFqoOnt=nw>$jD zRy!1%kHyLb!d5Z=^5i6f1l8ft=|~RyNGw#XUhew&bF7|PvEjr0ABB+1qL{_8sAEi~ zxI1*nZ;t%3DL}V43Br8N_F1#{vvUtv(WTTg@g$nJ_dX1`JkSu>Eg0gWFs#X%W2iBt+7b? zm=h;XL_&IVJlKX%$cPEMb?a7o{ip+%|J18c)vW7+5Z%gRjXg$##~|oW?XSYrFCb%J zGS%X~IfeZuHv2H7fq;Z6EL|YNviMe1SQt%n42O_G-p$SNry|CJ*d-||oA&&jIub4h zOqk(ETSu;(yu72=u3c+<@`T}=U1=oT*kj}zE#4+po-nemN!2)L0HSIE%y1#Z}vq*NK?0i&YB)yQJpj^8rq~dUcyq|ks^I6&q7nWfW z8M>`Yc0BLiy_>G`ipJ{ev(aUiomtZZc#X3B)jw_Y7Ut#y{w9TH0aACrFo?^@82G?4 ztLiW`e)*!ILqoU>@rmYw4g&{O=(CqEMJ~TM*EYVpzP@g5ZJq5gosn1(E5F`#jTedC z6pcvtm@nyxfJo9dr0M8bM2h#-)Y4)|6gBm0PgORw{Q8WC|HO&Z&D96zUNFG`=N@Bs&ngph z^K=2yKv-(gida28y-4elU*muKzz_T>jAA3h!LG+P)S{G9()(mL*O_3EQ$Z*FUfO5A4~i1?ovt>&`2tv3%~ z{PP7p7Aq5<=VzJ-ag_@d6oK4j2XYY8dgtAeWn5>HJ5prb^P9Jn^jftpU#3E=-0uLa zM4c%1X$TtAA>z(wFWTnZ%u}(nd|Nq7jsyk;Il`-;@jR58i+IHLhK%puGXN*MXUbh< zINauX1Jcve-D+zF?V5j;*Z@}3TAT0h@s$%1p+dv%N4r7tpt{b{&F+_8FHCH5x+7qz z)@FOCWm4M5y*i$srMQ`MROn`|1D9c?o3ORDwV2J%Q%E~wK4cG8J!99SsyeC{E(GtK zgLDgdV-t29+epaGQh&$=!?ftFk?GZ#X=QYKkm8!qgfUh-n!K{ z_dGZ>)MBv06~&E*o)4MP)z$Su&B4OQ+x5gxo<9BfYmWkN@jfo+ZjX79+1}svg>l8b zHTtiX68U9g5tbeHGM*}e3FZ*}X)e%0WDngbyZo*mhanj7?=X&a-7_3M}*7!BoefO11ALzhQRZeXx2y=S)Hbx6VY zug7lR(XZJ#IkJ03dSP=t1*y&BcTV35eOiBSrYq|jz%QlZ@gU+OD;{mloZggmb#*OY zzp8q9$^2=1(PL}W3+2p10D`U4WW2}a!5s6JGyzDQ^7}yn2LSsTXZcH1*#V>j=^?!tBrmBoVDs;e+RBxDQioS zWx^-btNiVH|=hgbhCeKrvq&s(uFR_*N@vj0y*9 zhk~ULYAz~#c@yu5k@e)hUXIyQ5- zdqkzq-B0!3mwuz0CEJjy%^T;jtBXbUb~Z_nSjTh#!VOT`4H&=CPT5=iW?gbM>-xg& zliIPVdlEbFo?^DWVwu+kq0&ZSI0~CLYcC*ax><4|HAuUS%Z5$KI%ik~0RgwC_K%-j zi7@vOKyx7)8K2M~Xby#leyJUYUHR{)j3j=gs)7k0J*wn87QZ@I_Cxqa>xsLqPj7#J zb7>1;wD|q+$K@b2Ei?~!j7PV8{#*y%7`(ORQ3XJiSD(=<|(#?A9J}4awah?5H;_*Vt z(Y(^exy0vks={8?kiUq1vTg#~R$;rtb0K}t!fSzB=LDXu( ziyG<@r<}#nImg7woum}8+t{(Ou>p%EFN5>v3HEn}4|s)ya(=wBf>+NodH2*$VSYQw zccag$Y;U7K?&;GIYQt)oPc-)utId9deh`dLc;B?OdZIoOa}7? zklO+;zP?{V9IhJp=FJ18fT|ip)dQ8rm zD7=J3VGcQc!Q$d;+Qr%f1zla;f$`%)LPGK{-(3}W6c>llVYpplaJ#hsg>RCYn%Xg_ zgDi{7yL%EfGEhvq1k8<>OHRVmr(?qh`;z?H&mVSnc7k!8yJx-Tiw(1fyk_$l6I+?J%^I2fA1h& zU<}?R#7Lx>FVP5#_e_EHuh{i1Ggx?TGRYz9=CLbJghXE?NE>gRKGS_;kw-uvOjcB< z5-qOUpr*qh{yY7{4ghwPMH0uebMmd7djeP_EJvVfsRtBwn2G@M=vR4&|6QLAg$T8I zeL7E9fB}uijzBt$LOg8(XcLCMrEvuVpwG#5wk>dtn1kI(MaVQl4ggM%$;q*&`mB&S zIXf?d6IE7{h*r*>ZfI&UA8QB?3lFc0JEpd@(Qj`71wSbnnX0mJkS=+;HhE|nb1o9(dRoJi`AQE=U#MpQ+#mlra%Be^C|4DUAtFe zaOch)lvdKO^^=EE-#~5>To6qbD&WU4vJ3Y<(tnK;QQ$EduErxH3%GHx?SeY zb)fuK4kAW_gMNieR_{J9e0Ibb^rHSb_gu zjNB2i$X5o&bz|^x{P%JdcTmg@tp}DDNg(ptdH8svYd_p#>4o z`4F>~M?CX>dHG({uhnAFm0E_%raY88icK>A+T?M?pW~}BC)4CM2b|4SR00L!)a9N& z#B|O$@u9^ziiQOfG#p3p0b>wY=S;r8%KG^K<^zXEicrU7|hG}HzYH4cHqjZi=K~H4*lo8|aZ)#je0Z@vBJbV}i zB-hx|Xno_s-td9El{Qo)-WzkY@3dd=6O)icvPNI5no5ssc+mf`WNY(M}aX-)^!qz1Qu^E0w5XQH@N_2!dIy$BUvHb3GiXf+)M`8E#*RNk+ zK+Nyo&)VtznP&*)*Tm;|Lmz7I&4r@{l?yWNzMnsT9s~Cju5h^Iql=-iH4@bJvY#`5wbaGt@4j(r!c#wf$nN>1 zO1qG_4SZ9io&Ovf2I{Xy;JUTC02yg%MQs7OQl5A*J1dL!?&+j0paai%``IXR^>}u; zK|b?3b!5Upsv($ysp8;eEW5PsE&!3Is^b9>1BE~Rqwe$JEgum&*Uq=vFP1vbQv-Qvr=+S%ct}A2 zd;lo>X<@KZp7HTp{OGJO4%m?gLj^Z;Said7Nez*383CX!VWmA*+az^v+z4J;a`^gE zo(<)m>bbUa%gV~mE!y8dO+vdZ0*|4MYyN3193C5M?>KvrcSZX@(g}q!J=BFn%rTfb z!q9rCZ+IbPJ5X>!Y~$jI;02DFmWylHR(TsYT5^=MvJN)Ue38^*42P_N?;Jpm-Z7Q8 zKi*xj-<&%XO=LYa^Qj%>U{yp#*?^UN7s+lyr3; z?|946Y_}bM1moNW#%m$E9x%h>$B!M*^Z^Ww;^Zno7gxFMiFg4K5j*FCUvGhA0zP!j zOCA-F;$lZOOH;M5Mf#_DtGy-Pzkff_3~U|zT?nkQTgo`fuz)7eWw|BsqD_pMl2a0_ z!D~9sgee5l6$l@3adFOOsL$KWE98;6fpU5hYR(2ANG8v5mEQZt3t3`Q_~gl~Yp$qm zJ%daM@$G+s_Z4@Ux(XD^$L+EcFFmkc)kD5Q0s>R7$Wd*pBF0n;8y*z}xa9)4U-R$4QCjGX5Cjw+HoB!DFm8J_4+=Ek$6cLe; zIzZUW>t<5iKf~J4*ytF$adYbeLWl!y{xHBxSW=#rkXQ!%UJuT>Fi?)*`iiZNLC%R} z6khQd3gz-Uq-uG2Ke64Vr;THhW~Z9?^9L$IUO-Fa!GU=f6)rPe&_BiaBr)-rq$CsQ zNCB0-&TPB@3e-*k0d4V!y_iU&0 z&0Kv2%|?ovIqQ?9od;}Yux*vNYr49qBDOny#UBOcfE$7~hy90Ne?*I4 zz*V6jcnsJ00u{eYa5JYw)}OsgnG#ps*<3rpQ}CX%!^fZ5)$&DYD-SQZz`4qLq0*SY zt)}~dC>4PMD>MrE?ft#|>%)Ce=bRM(fMYeIz~-1*j9G8t?Y8#i4*z|6wCGgW-?Bh8 z2Jn=vKQHXtw|^DX{1rr%mTK?KG2oRO?vS@nnm+73{Kp2JqH1~Z_{VKU$JboOg#_36jQvas=D~WE^ ziZgz;n7QC|mFv&W@{^B&_DnXB)xp4WV92t;4P`g#ce*W!*vZXC?#}+IA zgx+6g_p0X1$jAUr!i6hdBZFDqYfaj;_P$LzH)&XO^o3iul6HE5Di?PgQw6_AB@a}m zic3u$Uf5I{j1|UJKhot_*0@6>!o?ohO`^H9X-{Q-hl}I!vA`;0R@ykz_)C0cj#}wG zCY3ovM6Mm1O?}0IM9R4O(YdCf8edtv>h+tbB#Dy35DT9Q8-_qNRbpuUYPFsbZ=!xO z5sQe-aRq;U;?R}!pv=EdLBGQyiWD|@l$PM~m%L-&zkg4=DzF5Qs4VtY-p#pRB@}?* z=4C=b^nqmUJw}QE^#&;S-T9Ja05A_g$!!|m-;e~zM-1LB6`F$QF5q2ol6L}_-k6&8oG#j;6Tj_-Ja6Wa^hIs;V4T|A|&zmxDR| zM;)ZZ?sambG>OMT`DB&x<#!)kA5^aH^y?K86(;I_NghYq&dDMp*`vz{?t%1ClN?AT zyRBK*-l5*WF6%}~LdA|py>DQ^Xuzpect|9=f(0z`o0a6=dzlk{QQBmoTD)m#*%+c5 z>qDoa;qHg3ymj9*G{r$m(@K`2MF#)?E)MgsPrNu7)rw!!&2C~;itp~jGUqm5NZ1;p;;TX$c90BUtXI|cn)#3 zMG+05Uy2P7jG4-F3B*qJ7N&g=0(SapBhssxHPJ>ek~twJaEI;*>(`bnF|V96kb+kn zA%b=F#VaK@zhNG1F0c9VF@BDiAHAy{?n!KUDJnFs%fw&`6P=o9_#TGr+^pjMHt z?(XtMcPVx>@}b%>Q1t)Fr8}U4U=WDi54lO7$+%`WZ<~7lJ_?B*OpXP$A*kEhb=d0M zi)Bc6+>q443DKGknCOY+-3-md^ed?fo%hb5`n=Aq&nAWKX2@LQle`ZIRZI`BD&P6M zzL4`5i0z+TRGml!7Q4FfPlBNusB9{wzuemH};nDuuu^9O{<`yB!B3Ytugv1RxL;Jv^sQQC7Ome*(EJ9N2J+{oU<_ z9-r}O$t55o$Iw=T{kn|`FKb(ZL zw<~fQ+8*KzHIgNR>X5vf6xAlN!1{8IQNJvTwN43V~vBs9x*7MfZf*T8ik+5mb zKx;JC`qXRT`=>yLl1LY~<`B17xR9WZK zd=m+t{|s0@x$Bfve00b<@v&Copw-gEWlO=1MyL~!2%l|Lc|nNo_K>W-CFy_r~A zW+o~9iOXx57Qp|&MJ>{^98l28?qZaTjFNXssM)2!Ofj%QAaM8k?f~QCxH56c-52)5 zs?_Ax@_5H~MOVefUbeEyE9={eW=s$-hF7w07fGi*5^trab$x0e(l~1M&U3z`UlU{w zV4_hK96)I#iY`#vs}S^k1ZO0&=~2Z^D$MCDn4X;@&ar$++4QG=E4-Vap~pq>`u z{$gn%2><+L-idT7$`H~jlV4{Q?$x^*^tEg0rATa8%OYuD=YpNSJv|`?-m0M?oPHVA zBc!ESAf5Sl%o?G^A?(}RXIKaGaJ9QD6tF_Xl+3u!&a2SJF%B+Lhl;3vLuh}^*RNU# zPURIcGH0?72CUO3yVBRWxs=d>U{|7`SLV76)tKy>%tUzG#cF+<{`y62I0LWcvjRkdvQ|xvns4PX6ZLBEAiTe1E-_CPQlQ*aa?duj$oTCpt3jcTmXc@( zmg8r>#1SAt-4y?T6xIyQNlQ?=Sxj}lLK$qR1Z1IGiU7HDnq|mmRl5p!cJPJfvpq&X z7J3eVrxf?zu$6-@AG96FvOPK6xYF$v1WyEqwj`l|7mDM6(fUve5IDC$NboaROh}fZ zD5J0NW{mr0TVJ{p!A*;MaeC)WrGNit@`3kvNa$RwcUQI=jraems_y6KC8baF!wnBm ze=?#_mQ}@nR}mWNB%J&%7JT+dX0F8My$4V)OjJS!5>L90O}dUC&urV8F(H`~Q*6K4 zb1Q2LeJz9`3x>fJip@zLVIYeC+oMDETF`o#8?kyb>NTp7Y zN}VZ;@rc-|Z++OZRAIDrv+FV#KH26Z^EDx19Lw^nN2E%(@>nv6uU&Zxxj?dvJLF*W z>;pC!@|~O-McPH4^9Rpp5d0$1S@|1`fLa|M=-h-xUw>y@y#1UzJ>ICl zuH+|{R4Oq@+CVpwAlS?3oYcp68pn87Jaw}epfLqlWF(n_mm+ZFcXp-(ZwA<##{I3xv-$BfYIB ze?PD%Avt4A{+2jhV5L-#H=PRlD1jV9>$Glbpm@57phd=TzdZo*zs(hj5qud@|DeAMBtaat1v2;hI2|U!d5##1)Ei?8asTXlj#!Cn#Dx59LJ2^n zdx2DlL*Z5G(4a_DXq45NWk!5)?;Uy74TW*j;&oTQ9M3%4|48rtT-JAZS_`b&U);tA z!# zKbG{&5!}B={sFr_RB^N9Un)<&MSju0{Z>8fbp{XgW391dVp|)vG|fozKn;od? zLe1N^#1Nj&8yT@b(cnWP6qKgxS4%jK#a9?09fL%a$oVnpL(FP`cD@?l1?5P;&T3W5lHOH~=5-l~CJ|Fp_k1AEsjI8wp&kIY@T%Ezb8yF$z9yoP zlAwAd5r9HuTR(cl6!6!8=VS`n5r~Wox(_!3cfSmzHA<%0Z*0XX{3Qj^0!??|1PC`D zU&aZ|1)8L;KK=foLHPlRfXFqF`~z)-Xu$^L9)915Y>V^@y*KRu$(F}nI!*_F-v;a? ztS|}uUp*vSar+^;hK2@|=SR`Q+=jDg@z@cFFFDn|#&~omx4=&BbI5*QBm2*?|4;Jm zs;=($F=$S7S^@#H&aTt~wAig?#r*)N$BsgGmQPNB#km)^LA|t-Z#+v}2wT`Gi!jLi zJR)U?fp-rx+T}%FQUmD;rb5MgD9O)Qg7#UvH*fC&{e6bM?V!q}bz2XXf>z3zqc>xX znZsB31+i5Il=HEp8|OC@MhqGyAR<9&&)E$8J1rYq|Clq#Yn_nA)H8H9i$Fnn_2vyB zV`N6-q}*}?edp}@f+67Rp6YOC2VnvW`^i&gX`cf-lSx%zk7)zz2CYcwT+Z3{jwt+b zkSYZ3E{aRBrg0K2$e=S5_xe(rjN2SDJOwIp1D_b(EY*AWGMn?>l~v&K=97siVblIm zdgXk$yJ<&GU}zA=Q~D^t8i9^TR2xIv#i0M7!ios`>ezrAdAPF-atbvlJg6!((pzMq zU+%<*_J)C=;5jS&;gZO7FGSNRE;h>Y>$sY_h#%cY25$fXM!U^idyCleXYMTlQPE}? zKm74yIlprR94F`uf<{ZNGKekH$ zS*p+>yZSj}(j(-@xanf^dt;(!?mDcTDd!4hH6ehkkjZcF4H|B1GZogxs}q*7?#X;V zBG({K|BN1uA}!&V&d@S9g8PeZ4odMH%2Kos;Zs%!Iy9c+T4shMPXk$)*|1rvN(tWr zd8@N3Aqj!2e?q3t8__G;a!g@weomoRMn>?{&ldzD;a(p~h>US5YCxaTUTU`+@g*8K z9225PgpDN4i|RcziOT7aAiW)oP&x8q3DT~_UFs+xa)5FgDLnYV8AGLz2qW!M!${IU zu)ReziJTmi2}Xf3#Yd!%x(@0b8lDD%rjsR|dUFL1f=$Rd5t3EeW(*_-B02bG%z*gq zF$(3%vL?)ayCa&cF|p{YG6sHhE2LnCAh5I<_*^6xo)hLn3lDM>CHPTy3y)7?#Md_x zKYOr|f9k(q$1xfsmnt&y#^LOzGfw*RyfKVDZm#!X%!%{xT4B3#i2)es#BFyWaApUN zT}R0RS>w^|IW5MKOf+{ow}{Py>)*`Ipfh77@KA_X!buo@npbSgFoOwTSx}p~nzJyE zXiEs^un1-r^cz5oC%fqatmjpZOj-5JMq4 zom~22oN-8H6UIps3h#f{4H)AmrO60KoH*>Qv!H8olVhP`=F< zgaSOwhZHqNG={LfmQ70i}l(Ar9lMGwFpmU1_-?Le}rb#VR*R3fG{p)3d>2uhD|WS z27L~R|Lto0&zneLtYfZ=BtH9x0qYnfD)&D(-7nnwkFlew;&(8WcYt?5+;!?CH6dbVC4yo(owVD?nz+;b$z}wLZ&8p@=fjfQuwK>w(RnljTq-Ws z48ArNC#G=_A(cO_znU`%JTOES~ zX1;zDkv;*lU$=vi-{vp|x`p>-#WBAP8RRkVIJrQf1w)e_TAxnciYd@8Vh4*Z5hIxO zM?x#U4BTO11jRz>I6OMC1yY$!!HhJWBgXp@ofYe!Gj=n9iO-}QjzO8$#-qYe$xpZu zO{sFH5X<<2faLOQ9Mwd4P31&Seu};@KTp+iZ*6B(#sbYx$X2hK%!x@(_P~z87 z(1Rs|k7x<&2FZIfBm&q7Mi#1OL^w>*!%fwwi_ZWe2^v$OO@H$0r)~Wx@ASxC}bDyG;IFdTYDpe{OT?AYAx_;zAz-LS&?e9YwgjD^_3^^L1R7(C)L_rcY87KPHPxPxvDg4z0XZ8`EfA<(3esM>9Du05VWNL;uf9|_8SWRaU zj2|?SzlY;69@h3&HU$uQcr@z(JTC|}ofjP`%4OwOS^pd88Vr;v`!v9jg@KJc&mack z70DrdX%PiuaVIr+_RS4;5}k@8Wu7#kUOIkgv0XI!fFEmZ+V=f1pb9wzio5jip_4G- zpt1Kx1$M;V^M-NFPuSG7o?NKK@`KZiiJL-{-o@45D;igNRbyxZ4L)I2p_opR&l!Wo zG#@m7IN}c3!4*YxQkAF^Y}E`f!A@&2g~_BXa1xLrOm7^~fE8D3j;twvw;Ld9$nbX| zg&@~uIM?g{{NGey`KTXD<`eXC$Oe?DAIByJ0mNKE-&I|3exFeZH;YEyZ6R2$?=pywZF0iBOT3$hw*lUO$n7E z8v}vXIEEfz(&CNd2092RH2ZZ=XdFe4ap~&P<(L0qWtVZuZ0iyKFDcdD(Urj6ih>)_ znqeYSMq-|JrGGV>`Pl<5m;Q3}KN4&8EeaE8M%U2LFz{)I9#V8tqd-11TBZ6D17stk z=SQi*yqfJ1b!ZkpTBM^HMW0ZjqX31; z#DfPV7~nJlTF2)pOopF$i8Cp1ABK)VuR`-H?>Y4;f8BNRI*yo09A>X=qFPcM^z%H@ z$be%S>ry5L1`)lzy>2ORa>in!B~A)Bshn6H!2~!uv|i?^dBqh&fPc~y&TSa0hyT_l z9D_!UJ;0rU4CE-i$9B%v0qw2^L5-j(mTNfQ86*^TDF*~<&%qm(iB8(s>GnOG@kPE+ z9(A4z8Q%iy8;967(YRq;FN)5xij3l!VyN=$`Df^}PG+z7z$w<(fJDW(LW z!%ziio|ir=m*8lh8K|VdfkbOSOo0+xveh+EJuEkO5aX=z#d zX*flr$5dECQh~#aeAKzCDDMy zwmjcyQr}N!3x8eMeSWR)SOI8;s)Iq8PPNa*vH`u7{*-Bv-qOqq}}r-fJc3i`zhw z6WBDt%lD$S9sP%}p6sjkMa4%!mS|Cfuq2>|%zeJZjewN!_DDDpk}G}v`|ASj!zG`^ zETI4%>a#)reUNLhDVIZ0R{-VOBsbSR{!&U}{)xl@$8vLZcEoINR9D|tz8{NPBV}?O zDFeM2{}5;BJ{jUSX2}W$tQl#coj$p>f_%W@Zad3t6<#GVp2OO?2`C8dfwD?4mJ z^hDk=5Qi+Vvm1b>7W4h%wDxmMleC}6*EP^wqwuB4F z$Nyej^Ed);?c!H8ghhic)K8qyVu1D^)6$Mo(p9=IqQb-+ytZVIlu`~;69!0&8FR|6QGLZcgu^M_lbDOw;L@8J$01mDqsNm<=cQ+A(e5$V>6b|Zo4?K>ysE6?x zF;?;;1lHYJXF>90zs@87jL=(Fye+kzefQPJz;hEGoGA!{a(TYxMijyTY8cs~#Z zvJ~Z|POlQQqsjV34*@r-Qu{#96e=YkW^h`y?C(qt^fFc(-CBJ6MA~1$GRK|3EK?&U zSNWU1YZ~G3obVm=y!p)Usv#nuu?8|q)-$ANY>hZ7RA}BvKH%2>0Bu_b^?!PP#p~=1 z(P=C(F6mgZ@#gTJNkcUX3B9D@UC~2U(9S9#Ab@`zmUFTee5E76-j zcT!jgruG<>g0%vt&!@yI*Yp`q{kBFygD|WRoMI>`Y3qs2$;k;)`33^=U%G@UN5Hn< zge-^K*=ecq*!gS+i=1PxE<&VhRxIn+)y+eBKy_9NH;@vLM=4ISJ=fSy$khi+ul^+4l-KhRvSRO`351WxE( zS!JTlOUs=o<4!PrWE|n6!{|wV#QGJ-4&q@7)d@X`0dUH13uG@xF!|SshG75h^h#p6N4~dXSs2v|k(LPahvXb)bI^)!txuz4aRu8J^5T;Z5oqzVwhR=fo zn}X_}gTJ@VWM~D-c=!t|C1Q$rm>HNNjx`eJ%3KXJY+8q7_3&U9#sWe-w5@!cbqwTo zaptQjD4iqz-u0DsSxVOyFS$d0s4RQ&z*TRBeY&LclknB|f6jLw8(SWG{<;_P*N^UX zb;=(ehuo?4r`dR=OlL0(3bW*!(T-W|t{k<4dJ~81GGz-HqP?2SK~$fzB5U>{9~P+$U()PJj7X;GJ{|fP5H#vNF_%&D3cbu)&J- z-x~S1*e4+!^{_P~6$6i_lQC6)%*a5~7S%&{kO=xb4#fKwlNbUXVXm$saLSG3=A88W zyc4=l0E5Rm10UraR=6IxhwE_rp>HKL^4>U^60&BOAv&S*>)1=#y^_2_h{3&JJrR^U zU`pv*-QC&^aR;8tb@hw4JHnpyrXJr*^XvhYPjNwK|8MvXWib@f9v};4d;qftO}!U@{PfII(*Ky#hcZu5oC` z5MxCdeI7hZ@HLU?g2KOL!cxH%G=z2UAYr|s8<6gkKIZZH4|}h-XAU1)#X!MNUsK5J za!oQuJMi)`HNPA3KiN+$yGg?iz(4_iCHWTRFG(y2G4*rHClQ>xzryGFQLvd84MAF} zR57#5;v&{9NkV}|BZ-iNe7)E0{r7*1K?#d7DioD1lqZZ%<8acZ$odf6*a1N zLmR`Z+}zx@NdP0GacZ1(pt5^seqR`GMJECgFkeAG5Q5UbX(U2(WDT@jpr;7L-4~4D z>;cNp2VZ<$r^e;mHRC+z_TPP7aOB#a`ewhpk1sDNvKC8LxAw{r!3F2Uc$Bk(zZ^_3 zVLIX5-sZv}oaS}nidR35p6Uf8Xy`kH)(1d>55QS6c^%8r)$Yc$j*Z0o1Q4Ekhz6bv zx>1OLr@AIXBWpwiIpj~^?mgIJ!^4%K|2VRv54tlIc9+h>;hHyt|4q7AhuetXGwr8Z zu>DM{T;{7nj#vq$wJr1qJcVFWx*Uo0>Xej}Y{4vjAqviqS>j9fPD3MV0_M6`=+QK_{)Ui z|97P}w+ttd8{o%N3gM?hHs_%Or3vi}fd=A|U+>>t7X-XG<2b9mdz()ET4{{tMe92> z08TwNC!d~|KD6c6ej=BI|D*2OKht1Kv+qC zbg8q9qi|S@&~dvzigJytC#~=;=8C_2%fHnI&RB7r8nndY`X}paEsun%A~)wwHe?et z4v9e2Y_Wn^!QyX7{x<8|=g~5EYefWadf<&@5XUD9!QMx^-TnC0r;~e)t=@Z(;uqv*Uc7RR5}@Nwp!%m^LO^7eImLc zoDZ)tphCpXY@r;+S|n8+y&feYQdPJ@8k6wTs~jH3I}GXBN@cf-sPpc|7K7F_7g2Fr z!1lpV5OL%|v?m(okHUhk`@tnG!&cbT`B%C7%<=3Y2E|7=!x;(`JF~PG(OUyPFuT~~b8^NWBz3eC0`dp$mU{bw&Kn1f&^V~|~&sR&nCD@pSw zbN3N{sPFa3G)@d(g2VP0A}IJ=M1`g z47!Ld(GyKRkJUeM&&1DA-K-4rRv3p+~ z-1Ct3X35pcH#gv<7aFXY7^e#^@@?yDDQwNTIQ?Ru%zO%r1s^bjeK0{iOaxT?rAu z`tK|%S8c9d`ZtsX4D~FBsIdJ`P*DFX&qwNna=zQUAcgnVb*fe@&Z)*J0^(lr z2V-fyex$-rJ3$@$(OX3!=3$sy9XCi&6qieE`7s))+~U4r2j7G$QwWM&o@uFuGk!Vlk^ zcx&ERP2VYbn)HGfy`9&h-l_hi#d)+Jqn&~PQijkQb<%4D$mOTSN0)q#TB6AWhQM}r@^wn`4fJG=lwnn;m8lu zE2AH&-O{Rl&K(Z_D;guF)DV(Q^XADXYo<8U8;kQ!KF^`o)ZbA?;hB6>`E@OmqxpW` zLT;&JWuWSyvJeJ-4#{1J`$UjSv?g)m$ct_IZ<2zfAjH}E*vm57%hmZl3=hlfu|O!f zeUC}AyvfP9Ks4U(N$=Pe=F!W+YqBrSR;>6d;B?UEepJmWiNJ{A&ZJO&x{1%HPk%hV ze(TuM%6fg;Q_bbpa1Ep7CPv$@BCBqwW+`lrP>SW-jSP@qCZ5&+6dodeX=}gc5;HSf zpvG$=XT_ml|9t5$sS*%0WniNG=dK5vC47}?5^%TJBThPvDB)q`Z=n^7h0*7&vAJCjef(dNm~SZjkEe^s;vV zCy*&rp``t{;u%HQ*n?WkG7_^_xrPr+9yZqwQNM1CnWEtvs=3Q>`dWQ8xU3F-wR>?f zzkv~tO#EVLFn7lPp?P&v{SjT3vP+o6nxl_d50qVUH7SZ@)8Fb)_TUd+?O61BbKEr( zlKmlw3`215<)!~s+gC?bxqa1@d{t|@cMvM%plFw6UD-D`215h~u;*+I52D!45YYfBe1Mz4}%Su3;~ zv6AUn?ybV7=|N56|2@Vq@yewYIWUaZBC+9Kmj}4u{ao?hm`ic(uBT@G_HyLq@TEUG zG0k>p%$`t6^Rkp^Zo%W_BOky!y+dUJgW<|sYHD||A9MdeOTD)3ewy;uVGz3i-ro$Y z?m1HhitAvM&;6dzW=&_|W2e87sM?eEi(BP#=+klzwte8}pUvqI1!pfLU1Ev*7y<(s zJ>N(lr;{aoCrxis-}fI=B78k}JNl|m&Ojbp*om)o2K|kDhusM_DWp_As!$o3^FGEy}+Jm#u zm0E~Qhvvt`?rtLRp|OL`PzK}A_LlQ>RXDmt*vWHK)H}>{yYAOa*6zy+-JEwL@c-#J zJ9tg6*KNN4Ot94AGIq!u5R2A&E#U2eT268L8|&3KOQEuO!}zU&-dwSBKbP0vv>!9) zocR%0>zxA(G|;p1(pAwj z!x6+AD)Rq2pA{Cg`zETzjoQt&hTKN4M^HkX$VRtENbDx1JI~WyRyKz?nV_I18Z1k$ zTrlh?#uEwbEO*>ddZ&m2$PHWYw2lSXcnk`jhKU}%Z&TAC0sA>YJgS2kW*iNtlWnS8EyOLT-^rMco>rUQesbAPX z@nMUux<&)4zhs%m40WxTQOV#x-Q=OEzWX|IzN?XomFCK9`Lc_<^Yh!?u&YCUU`Uvo z#F;%DLj}hlYsag6emEOcqn&hCe6S;vDF-Te+G2z2+f?MrzE?01wjD20In(g(04RDS8z&20!WR&QB3o zVI`XDLael@OHM)(rnX(LL+MdNNl-DeC7bES?FL=#d2Eg4sXe&02J4u}fH9h4+% za$wjSfztjRy`d*vA#SWC74>id5XqqiE=RFKjCXOZeT{)^ahnksjQFF3!zwK?IVa*& zFX{TKkXkvGc!z&@MHj<3%zC=v;!Yz4)tmB)sDFo=>eL-DP$R5r8fr;S^>@qjhvMOe z-s$kYOO64gQh4G7_bm?#(xUtnuXyeG zr&l%o>-y0oI!i|P`RT6=p{QRzRv^XJr{f1!<(qfWA%=5*XnO!5I5p3O`a^)3HsQNA z6+ziXt#mOt)Hez8u`y?CcV;Cj(JnpTY*uR^u3+=Apx|~!p+eJi<2NYA*WG-#?>0{D z9qfQ~T?0EsvYt-VJmIfoRUC~=^ih^Y6*J~-)ArqxGC6(sI$y{K*bv&!m$1wV@lfGQ zi2PFo;HL@m^n|W_th!3}xSH_Q?(P#ol(#%fV6Z8xpbAg_=--iH2a}7A`P}&`EWseV`*DyVg(KfTtLNZihtR_L;b)6n6jdP_ymE2|(nuJN!t1+lkdelW-=ZJdPQZRht>1=qUo9M$ z=2t>+VfwgW!Biv=!ytHiI_#^$YZ@7=gAMry;RpbXyqT1i*FGK(*O~y`=oz8ZZ3ccM z=~Hi7m75=abXcQ4|As|_i8LV&SBdqS$m~6?Ia0C~?4_flDF^|q+A^{YoP4A1_oK$2 zl#3X<^X# zPVvKCK0}+kWbtd;37waupB@CqWF4$lxt@)q95tz1_s@LxhVWolvgEg9MfiET0UgtCafVEY;s}! zBzID=$(i!|`q%KxiK!*Eo=hy-KFn68@r2C-G@+(Hvpvsoouti4+Ipwd#KfgPGr9ld zjh}pcC+4MxiOF-SIdf=KgiE9n!~3xbZD}<9gcNkkDT^qDF|QkFL-v&0nHYZN1 ziw+uEF6SnRBvH{+a7WnJLMqfgu4x#6*60< z`ATf4iFlP@HGyup8UD64ky>ou+};s=qKa)gDMLVUqG&DOiT5kXxy_J`&1D&8$Z@** z<7k#a*F;HG6(523kWmnrh~dBK4GJ{~vlV1CF<>P0*Dp_GCIA7uCU+*5FO>LaJn&AV z`v9Ruo2L+fbm`X42M;UhGC|<5Ar6q89lBoN0|~OJUI2AVvu40zZxbav@Zn=}eVKk0 z&gFaD!N%hZ0ERnP4O+NLA#sY=Bv$ZLdC&g~p{$RlwI7;XW4BtU8u{3zCG9MxY~(pmx-l|?>Z9v-EOWj-OKLOw}cXvoK7}(<#LszY3#m< zN7R!H_#j`|G5=`($g6I^c&v zVL&A3Dxgz;SmR;(ZR%R}9qU^0A1CYdPLet?7Fzm9Z8%7))g>;(rBkdVpFtmEMNbq< z*j;kd3JROg)?Kp!Wdu_kUU&xYU4Ji3l4t@y&jG))6QlFkYx-Yk*mo(0>wgtP@VIF~ zsdx$#@^6NQ20%3Rli?*860{P{p~H3&q?NcWcHjW>yUTwXnKR_|-Q(30y2dG}#2|ZF z-|!2+PdR*lW%xd2G@kBtpiAnxL`;1P)?i~1F8OT|!LReK5UC~@>fh@?SRL3BHhoSO zXIVrfPNk$jZp@NOK>KWTpScrq0!CczuS0W&vGh;3Vn}r4Ii6a}YT$Y2lnsoem;eCh zTz{a&nri8!CI+LJ%OMq>qt7djMT!aTBtEs-+#^^qpodE!SpJqB>6U8eDo$d$kpN2uN|;2J zV&&Js)(5OnZqr`E%S4;{X3}p*d5a<>kGR#0)ji z+(705flKsYZx0!;2V&a7-}--KI&?1@sCSG-UuMGX3(sx+8oJ>}`0KzB=G=hSU$bq? zaXED11>Pb-aP4&B)y(`fRkx8~`wD3wIHwYqU!?5aLCKE^6WJFy8nULcXs|1Lbx=Xp z{42&@y35#!B#kbMKLX!s2*v$WEt#1jv{wgFOmNo#E{ynKrI<>$M8onY9UYkOUtnc*>9%*t`G6#o-yW-^AgA> zV4#%ZX+sdngpqv3N?hRm;TP2f%z0B9jS}#T5;U}pX;3OUeJVwpM|fOF;W;aSl3f0w zO>?aDC$FFIEdunk30_fZR?b=3QWD}@S^TCttZMgA71tW*Q0l)p#n3Ehj-MGqvof5t z%5lU{@)y(%#YZnxo%ypo##2ecm-ExWb9RvWHhp+ipT9Rg&kYz-6*rBD3Z@cYm9P-0 zkFvJCqK3UGW?C@_`HnkZyr40Y;MA7AMC(*ba|4!$eZXMq6DV6E4KBp)eGmzQr}QoL zEdWI6K>q`}2txUSt&kz^!HHVCoJ`ChY6)Yx+c%SX7k|($D|@~MiEo@zGFP$S(=tEk z?2SX1Sw@3@!C9eIj*q%=%4;qI1iG9x=hQV8TC2mc5UK zv##7=6EY1WQ*EW@nvc$pwkc|u^v)+YjKV9Q!zllVQEnFA|Fu~?_T$5AGfXuiJI|LY zWP|mWverA;n+Sdz9AupA2|!2+hrG1y7O@-SR$Is>u37kjOa~%G2nvkIm?!khmY~_#3z})| z2h?vas=`tn#PvZ^%{?wTc;25ic-%Rp%U1{)yf#F-&-Tu>V~}EE)_=rk(U;XHG;6n# zBk0@IhpW@-=bpO;1LKT!GIo|#UBgrhzynTVOQtqL`r8$yplFcEw3?B{jDyj&XuU>s z5>lJ@v9&3R%R#5YB{;7;QsjcVps2_!d;Hkw9iYXEGv<2`_l&&d|CH!mJH=#QGtHW+ zBE|V89ET@2OxJBx^gW6xt zBZpGqa%8EHxOAQI@Rx}@wZDDEok%Wx&JTSI-R0Z?oUF{q&L1rIZhON6|2R0b1wD?c z?jZ={G&xnJO_o3R2u7bdp|a#$^xbLEWJdmz#uoa^uSY)rB)SJtPLz>HAz3%pQRy;B zvsXaUbgOY~@hL_6&LD@fy`ZBLc@@JmU-MM|=#sU)lT3^ny<-eEi>2vM0SR;rREl;- z)e_qAa5KTyW#JGmiMS>PSXb?iD-R2LCYFe<(y8U7>aUvhD51D!Ah` zSKc)Fqj$Ie*x~5@)(6S@K&xooNFg38xJEZ#7b)Rp`#4@E^WqN-IvJsr1?HI{KR_KJ zQ{S?D$*A1zvMj6`Joc4sYwFA@?auPUbd2u}yme=XNEkD!LR_c5ff8#&U;WO20b@s? z`SOd{FzZUSf^!R5o`X1mX-;lt9p6c*5D)_BH|+epC1#7C41TVL*|;@N2IoJ=jaMVt zO!;vEwMdV%p&<$&20yNBf_x;yAe3gcs&E7LP52YkT7SBqRps0@u;mWWxQ~n>|5NLM z8nM{`D~n%o&i?AsZ23C_fEX{hkghn2Ug8^E-@pYJjFtWC?`QnwB363d;()HPpL`eO zt_sMx3x&oX_Mw1iGu*)_Yoq*em&SV@m5P(&&2Q?(8QB75%)-JV{1^p97hK)=>(q(h zrr8-ZeL@!I|MI1kZB8Rs38~J}!qE+oHp5iYk0viJXV5lul*g5a6WjH~h6({xZg|f$ zK7HjD`E$CLL+}n0Z6-b3V}~;HG~pM-Cmbt;O@{W~T!SfE_LK03;gaxyUWYogA9;|B zGYfdl3w&otgeD%_H4#iK8wIhYQ1AZD3Io`AX(EgF z^!YNQByCxWl?EiF&{iL)Y1VqsqKx(w0D&WHQP$LIH!<|1hLcx=PBS+b=`~d1EI0M| zfUXw}El}gIe6l&5INvtPvFn!oFTOyH9x+#3rsGoUKN}=B?`nF3m+VW_0ISh@(Bktq zng6M+A7A~mZeQwe=L{?Zc+YF64U20KQY)OhzOM)pMQ^`&%OKi@6%|K}5Y(V~Y2~j~ zo$)}&`qP%Z;-)?>0F&R5FRb-BmlQkw5{2nz2#tgeWvE1+2f@M3fOV5 z!Yn-P9@ie>$N|<&s>>ecI)n4pRSXR$%hE-Tn1pHrxwIkODJgAQf^BfzB19RydineG1pOUD zNB8q9U-Dber*YU|WGLSQ2-CUE1K|z@MROR#a6i7^bI+(C3BSJ*O=7^-R!9ONT7n75 zYV}8-32_Th>c*gA&|S&(txrMYC^3E1>Yh1O;K$Ow#4I&h4bfB2S4041O0GF#^W6p1 zmd&tSzQQoVw~}7Sgi1b5xMW4Qgi5!B{>b8qDh@Q}p}9pa7c9>yr%_Tis9>wr;t?l5 z;T;Znm+ezA1uY*O*(e1xV4={7${%&Lo9X>ddbhaR!>4AFQ4A0Spk(=|Ji(Lg=ubpy z4FO>mDV0P3N7rU+UryVHpxI>0oy7$KW!n1S)fhN&XX{wJjj{Bx9Bq1qquUJ@nl=Sb z7m;VwRHH@pyX(NDL)K$L49ilPGj!`ir91$#Kw*4)Rqc2^?Rc2EIx1GzXCRx|GQb$#A%%<`sUZYs zvJN(`yAx>+vzZqw>1ZO=1b5keNKa$4TgLFdaFdR_yD9}x69643xR;0b$yM9-%V4MH zetAPyZm?+Y#*e&2=(?qbyKlKn4q)v7=4$g@rX5k95RpMSASPko8BLc_<(ruox`e~< zp&o-q!-_6G{0T_>6?yv34}lB3vc7@( zM2$$}EUYxE1bVD0m-RtjIw=ZhUr^}~WSW7BD{g}+u57phC2MT(S6fBECsPLQz_Z6K z_n6J-wv8BZ3j4^~!zyv77aa{H6?CJ4yHB~UIhlTYuHpr72lgaH0Q0w}6oyTR%~J>{ zhzbA!;=*`5WO6D*dO7tRwYgM3z8c z!!8Pk)R=#eFPS{A8<4rAPkx#mIPh#y!6N}ZGfq5(LsmeAmx!tvxY`K0AS20$bfcq% zT*ArwG7dn%rA(Bd>m!b3*-~qadL@ZI%`zi_C<40(0Ms&LLgR*!(A3o->K5N@0!0x{ zV2R>qDey8Oz9PY3k>T1JmfDq zHIw>pk>@)j=re+r69%IakWQWq>AGheN`89GbA4ok*eNDg=^NVExZauC5829TB% zBRapnynWifKLU$QNR$Kfvv>Rqg(8`xM;QRCZYESngVWdCH<{5ui4WO{%wMD(l^c~I zpd7AF$j&dczod_-HxLtHD;e910c7DV8NCn6>A|?4nD3n063U3enBd1msL)4b zII+(pYBV`O!-a)xL>xjRka1yJ+kw6U@1m~dQK$dWl1@+)J( z&f+i3PfcduPB2NWG=91gBHwWm*IF_V4zy}o%C#1${uht#shdE_0&i^zZ5I7e;~#+1 zLWMk2_2cs>;7UL=2e~$2yWzu61%N@KzFx9<)KW<&;o1iro+8+ua)AaNlE8}K@pxP?gu5CtlDwB+37(<5G;LC6S*N{%pmC%|f9Dp%6>msgk@y=caSc5}XTHyy z0BIw1agqH1B8U>d{-V09eRA7};`b#yGDCi5=iEZd3fR+vLj0ICP^<80gym9uTZtPx zqf5K7Ar&A^CxSU6md>)*LM!8dgegGYo)s%`x)*Aj^hES2q#`wv=*!bCZUE&^hq9*>P zHs7azrN{DJ^hpK2Z<0JvagNc>jL;J4z-U0Qd zX3do1n?g)D56Gvv`|8!h9S?lNcgfUD@tFghi{C!nTnl*dh zjlc_nRW37*6;oRzD+c2QXcu|g==_fh6KVb5pfD8v7mHUR_`*`~8x5CM-lah8|T zP_m7}%&H06{{UuhGK?&3ncaRr3>YE_tTXJiVOz(hZX6>qZaozK9KuDq{1IkyQk1RYFo0X&rz?LNk`%?GU9;ntQ`oXSAnh zQ4_c%efP(r@eSC3zeQU~pd3pC1=Z>cLHF1C?)dPVlL6XVqK5lmSy%z+Z9po{#G>-D zgA6$8qX3(bMgm>$QZh4BXomLPrb^8>yDqq0T8kH=xm{NbXlnt&My)x0aw}c@&fpce zobvjl0}mbnQe-|yxBW3Y5;?Ch`_#066Yey zW9>(j4ymNA2_Q|;=U9{m;XN9X=z0Jo<8QIrU~&5-zn zwq{MA7=ZN>TU{W1<)x`6i*Vx}7&3@5^G9j$T7HYhJ}Sh97Ji+{H8o;R_!b>)<0MEP z@*L=@7^$NIM_4Ph#>L^Dy~@R#mxhnrsJsPK&}jW)Knzh?9{AGMop@;z&)mv-9-2e2 zRV$zqjf!3_^5S-4yf6W3OjVY_fHJ&aBbf3Rp{IGe7cJ@4JNTT{Y*%O(J3$&%Lw2@YTRTu8M~PNV{M75&a>f1iVoGwal^ z5`g?!U8P~?s`$hSq0ZashMamJu0ShKd{t@1kd|z2GS<-}8rlZeH>VBX z76Jbt1d=8lJ(vF7Vn_J&vmH1lirI7=eiVS5Tk3YIV3*rY`sHJKajqH3XAg+~G~#$_ zIlVAR%~kCg4t*wI)7@9BI^*S+#fz@a zSJwy>pqHeS-7rzD-ua#MR{}w(J|-X+lDm7_K>Y9CGeNk?kVpPeykKKlarA^s=WX^( z;z0s|v&T1gP|PzG=B9@)qsiV>mkk2h%*ek48@wg*U$VEMi9muC2x2tE@y^~%dTiGr zUj_I<%xrt=p!M7n#`TAqUoURIc(6^d1z9q;Q&uo5Ue{#W59;b7H36?%MDGN|zu*8# z3%YVe{wdlycnS5Z9ft$wV6ZK>;LTilxyw|Q;j`5vK|Paa0rvfA4wA#fFV z*d2{#T6)C`6#l6VG2yP3=pEvM)K*(En{1^Jpz*L@Ygk2nW>AbOaUJQh2bNPDv=>yc z*IGS7QSllM;$lro(9d;J^NnLQ=S*jj9+=|rf3@;qI+5$$hHK66@3mTy>$Q2}8d%k;iRFSMI$RkY-7Z02a zglSR~XVxZf4k@7WGMf4}ZT#w?)6FC^7&bL0rCuSj*&cZz_v^1#u@0i8qI7xPwXU5c zk5~NF^Q!GojURsb|OUn{#q$Jeo` z$Hqf*as{qCUm?cza7qjc9ggnHjdp;UFfgRifI|VBGuP1bJdyHs?)qf!B(*m&D~Kc^ zbFh^T)2r`yEY7?C{Z}mX{{^T(A5*OI&+ILhoGzhY_@+@_XzW(V3Dv~0we_Fj{i|YG zS~;iw=0+_csJ;CE^%akYNv>mI9ZC!oZdmvGbeK-l@o#zEWH!GX5)G7IaL~X&!I+$R z6(TDpQP$W<4L*=FcYPyRaNSS0FN2uLzor|qq~SkrCjHYpCkOozQV<-7pwdYQ_SNgx(}KGFG0daBhD_x7 zEO>oq@?~l@&uu(y!M5AtfFU6QOCn^w5WDpn8Zc@0-6$>2)#vRnQOBT1LZu#jTg*BSR^Fq!G!`SLXCqq z8k`hh#|KlPc;H^Xj2K^mg`qTaYEDiPFll59Mv(7DWDlG~@7#J#)wu_DP%tJS1r{

^E1@XO>HmLrB5r@+nGIZ@Qd4e=GPZ2B4t zvN#~3jf^(}b76D6X{jCtClQxo7MQ+wXrPCwd_-Vf>ab|lE7zGTW#Y=6Mh(Oe6s9aM zGfQ}?w8H?VVzw67lP56vbG;K0+}T4+OfND^aqrAD-O9Qd&^lRs>r!o1)zDc6h#cP0 zP*Wqn;WWwe7e3w+4vi^`R)%1AK<7t0iYOvFHoLobV-QcB+m$99N=izxhydS>uPL`? zT0Dv~Qc~DR7(_K5fv@lE;6q|RB?E)>jI69HAa%RF1jZ|USy_yr%os#@3%r~I5w|IB zu=Ux4aeK0$^D`gBXqg%@J&laSJ+cu1A3Ha&yjlXCNM7fuIz_}d5wVQ%U*`W^2nslI zh>vQ}@!|!c8#itvHV*v@3zmqvsH;rmL`W+GRxZt5JG)rWsPs6W0uNO%L0^K0+isK| z1P*Bw7|}B4r}f;i`9^wr?C)0H_ka!NDf10{57s{RayhTtuQ9WP!l0Ip*Xq z=oo{Fj;|>ph;@Vh=R;FMP;UV%%;rm}>FGFN%>}~ezh*P{tUxEB3+Off_}_JbAzdN} zh=A`|6{>4fR$AfPHYTQlq9P7r#YaPrf>GQBFgh|Xd)En;E672~5OZUTB%;Gaar5wm z$ma!X760^g`d3V8HtGJlM9WTr2UtzgAr>F7cp_~{6S-?ED=Q1xK3_pugz(2i=U*yB zXdNuXE&|5`9F>}bp75VlWSS#W!bvB0h3R0G9`aAG!#0fSw(J779|&lG3(8&(;njul zJ1#gVSZp$)U`j#cw1ev=VrcSJfFl?21P{5u+y!oH;4cY|2C|@jgsc`QP2!`( zNr1ey=!tOAHv#Zlcee()?D0xqOKy3&x(dHR*=6#)V7dGMH@OKLgUY3m0I z*JyZ$)ML=`iUBum8t{$WeLdfR{r2utGfpIonu59%EfrNihl)hr`V~u^6_M{el0G|> zi2E1ptY~-v4TKvH9WVp{54&e4LZE<>U-D(W&!zLsaJDn3)Aysu#H;4hHxPgN;W$Nu zFB;XCFJD?tRW5)-yt0OhitAOZdQfc<5*F4y1qG$0K%#0aFAzn(>c3gp)Q5aVf(ZE! zA3jimwfb$Brz^KnuY6BH*LyDMzP&vu*aDghUAcldlr4d$As;mjjei!`)vLGK;eetW zO!$##4x%(Q-~r`-Ja?Q_kHu0g_vtrCiT8uC9byj$Pbn91+4Y~@1@|(}L3&3C^v59nBE~}~SzliD6s;|~!s&(<=MT_x?O)rs5P0d@k-q&3*ep2-8 z8C}Qg*Bjryebck3o;cZSud8F5@EUV&it@XJIu_szMh$%J>kEIkS|=bNpwFA`>FHTG zH)kfE_;b*2!iy>fo%U%h$baqVBr3oZMJoaFbL_rS%>C9v@E_a>IALm>aMM=wRUtE6zPr} z9yWr#w(`7rb+M;J<>sa%Rh;7X_V#^8$GG!z)Tyrq3L{=R=TosmLs@UHg0!?W4=vHg z!h&oJT7JY{KV3}`{;b;E*xJ$nailWC^69Cmk?HB@FLH9I@UU%X4vS-QAIz8-8EH!~ z-)@>2`}3#u@aRa{M+bGR4t{ikvU#uw1|M^}2w=ZP)5RTXw){jx=!b?!9idyI@dCSoM E0U6xDL;wH) literal 0 HcmV?d00001