From 7d4564573e0361f6892fca9156bb1ee640620785 Mon Sep 17 00:00:00 2001 From: ModelHub XC Date: Tue, 16 Jun 2026 06:52:16 +0800 Subject: [PATCH] =?UTF-8?q?=E5=88=9D=E5=A7=8B=E5=8C=96=E9=A1=B9=E7=9B=AE?= =?UTF-8?q?=EF=BC=8C=E7=94=B1ModelHub=20XC=E7=A4=BE=E5=8C=BA=E6=8F=90?= =?UTF-8?q?=E4=BE=9B=E6=A8=A1=E5=9E=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Model: KKHYA/qwen3-1.7b-fft-coding Source: Original Platform --- .gitattributes | 36 + README.md | 61 + all_results.json | 8 + chat_template.jinja | 89 + config.json | 70 + generation_config.json | 12 + model.safetensors | 3 + qwen3_1.7b_coding_fft_20260429_153648.log | 219 ++ qwen3_1.7b_coding_fft_20260429_153746.log | 1343 ++++++++++ qwen3_1.7b_coding_fft_20260429_154000.log | 2823 ++++++++++++++++++++ qwen3_1.7b_coding_fft_20260429_170521.log | 2844 +++++++++++++++++++++ tokenizer.json | 3 + tokenizer_config.json | 30 + train_results.json | 8 + trainer_log.jsonl | 95 + trainer_state.json | 701 +++++ training_args.bin | 3 + training_loss.png | Bin 0 -> 40381 bytes 18 files changed, 8348 insertions(+) create mode 100644 .gitattributes create mode 100644 README.md create mode 100644 all_results.json create mode 100644 chat_template.jinja create mode 100644 config.json create mode 100644 generation_config.json create mode 100644 model.safetensors create mode 100644 qwen3_1.7b_coding_fft_20260429_153648.log create mode 100644 qwen3_1.7b_coding_fft_20260429_153746.log create mode 100644 qwen3_1.7b_coding_fft_20260429_154000.log create mode 100644 qwen3_1.7b_coding_fft_20260429_170521.log create mode 100644 tokenizer.json create mode 100644 tokenizer_config.json create mode 100644 train_results.json create mode 100644 trainer_log.jsonl create mode 100644 trainer_state.json create mode 100644 training_args.bin create mode 100644 training_loss.png diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..52373fe --- /dev/null +++ b/.gitattributes @@ -0,0 +1,36 @@ +*.7z filter=lfs diff=lfs merge=lfs -text +*.arrow filter=lfs diff=lfs merge=lfs -text +*.bin filter=lfs diff=lfs merge=lfs -text +*.bz2 filter=lfs diff=lfs merge=lfs -text +*.ckpt filter=lfs diff=lfs merge=lfs -text +*.ftz filter=lfs diff=lfs merge=lfs -text +*.gz filter=lfs diff=lfs merge=lfs -text +*.h5 filter=lfs diff=lfs merge=lfs -text +*.joblib filter=lfs diff=lfs merge=lfs -text +*.lfs.* filter=lfs diff=lfs merge=lfs -text +*.mlmodel filter=lfs diff=lfs merge=lfs -text +*.model filter=lfs diff=lfs merge=lfs -text +*.msgpack filter=lfs diff=lfs merge=lfs -text +*.npy filter=lfs diff=lfs merge=lfs -text +*.npz filter=lfs diff=lfs merge=lfs -text +*.onnx filter=lfs diff=lfs merge=lfs -text +*.ot filter=lfs diff=lfs merge=lfs -text +*.parquet filter=lfs diff=lfs merge=lfs -text +*.pb filter=lfs diff=lfs merge=lfs -text +*.pickle filter=lfs diff=lfs merge=lfs -text +*.pkl filter=lfs diff=lfs merge=lfs -text +*.pt filter=lfs diff=lfs merge=lfs -text +*.pth filter=lfs diff=lfs merge=lfs -text +*.rar filter=lfs diff=lfs merge=lfs -text +*.safetensors filter=lfs diff=lfs merge=lfs -text +saved_model/**/* filter=lfs diff=lfs merge=lfs -text +*.tar.* filter=lfs diff=lfs merge=lfs -text +*.tar filter=lfs diff=lfs merge=lfs -text +*.tflite filter=lfs diff=lfs merge=lfs -text +*.tgz filter=lfs diff=lfs merge=lfs -text +*.wasm filter=lfs diff=lfs merge=lfs -text +*.xz filter=lfs diff=lfs merge=lfs -text +*.zip filter=lfs diff=lfs merge=lfs -text +*.zst filter=lfs diff=lfs merge=lfs -text +*tfevents* filter=lfs diff=lfs merge=lfs -text +tokenizer.json filter=lfs diff=lfs merge=lfs -text diff --git a/README.md b/README.md new file mode 100644 index 0000000..40fc0b8 --- /dev/null +++ b/README.md @@ -0,0 +1,61 @@ +--- +library_name: transformers +license: apache-2.0 +base_model: Qwen/Qwen3-1.7B +tags: +- llama-factory +- full +- generated_from_trainer +model-index: +- name: qwen3-1.7b-fft-coding + results: [] +--- + + + +# qwen3-1.7b-fft-coding + +This model is a fine-tuned version of [Qwen/Qwen3-1.7B](https://huggingface.co/Qwen/Qwen3-1.7B) on the mft_tulu3_personas_code, the mft_evol_codealpaca and the mft_codefeedback datasets. + +## Model description + +More information needed + +## Intended uses & limitations + +More information needed + +## Training and evaluation data + +More information needed + +## Training procedure + +### Training hyperparameters + +The following hyperparameters were used during training: +- learning_rate: 1e-05 +- train_batch_size: 2 +- eval_batch_size: 8 +- seed: 42 +- distributed_type: multi-GPU +- num_devices: 8 +- gradient_accumulation_steps: 8 +- total_train_batch_size: 128 +- total_eval_batch_size: 64 +- optimizer: Use OptimizerNames.ADAMW_TORCH_FUSED with betas=(0.9,0.95) and epsilon=1e-08 and optimizer_args=No additional optimizer arguments +- lr_scheduler_type: cosine +- lr_scheduler_warmup_steps: 0.1 +- num_epochs: 4.0 + +### Training results + + + +### Framework versions + +- Transformers 5.2.0 +- Pytorch 2.10.0+cu130 +- Datasets 4.0.0 +- Tokenizers 0.22.2 diff --git a/all_results.json b/all_results.json new file mode 100644 index 0000000..d751c7c --- /dev/null +++ b/all_results.json @@ -0,0 +1,8 @@ +{ + "epoch": 4.0, + "total_flos": 7.974893111623025e+17, + "train_loss": 0.6604051219656112, + "train_runtime": 3155.399, + "train_samples_per_second": 38.03, + "train_steps_per_second": 0.298 +} \ No newline at end of file diff --git a/chat_template.jinja b/chat_template.jinja new file mode 100644 index 0000000..01be9b3 --- /dev/null +++ b/chat_template.jinja @@ -0,0 +1,89 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and message.content is string and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if message.content is string %} + {%- set content = message.content %} + {%- else %} + {%- set content = '' %} + {%- endif %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/config.json b/config.json new file mode 100644 index 0000000..e0b1310 --- /dev/null +++ b/config.json @@ -0,0 +1,70 @@ +{ + "architectures": [ + "Qwen3ForCausalLM" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "bos_token_id": null, + "dtype": "bfloat16", + "eos_token_id": 151645, + "head_dim": 128, + "hidden_act": "silu", + "hidden_size": 2048, + "initializer_range": 0.02, + "intermediate_size": 6144, + "layer_types": [ + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention" + ], + "masked_layers": null, + "max_position_embeddings": 40960, + "max_window_layers": 28, + "model_type": "qwen3", + "num_attention_heads": 16, + "num_hidden_layers": 28, + "num_key_value_heads": 8, + "pad_token_id": 151643, + "rms_norm_eps": 1e-06, + "rope_parameters": { + "rope_theta": 1000000, + "rope_type": "default" + }, + "sliding_window": null, + "sparsity_attn": null, + "sparsity_mlp": null, + "subnet_mode": null, + "subnet_type": null, + "threshold_attn": null, + "threshold_mlp": null, + "tie_word_embeddings": true, + "transformers_version": "5.2.0", + "use_cache": false, + "use_sliding_window": false, + "vocab_size": 151936 +} diff --git a/generation_config.json b/generation_config.json new file mode 100644 index 0000000..1701c94 --- /dev/null +++ b/generation_config.json @@ -0,0 +1,12 @@ +{ + "do_sample": true, + "eos_token_id": [ + 151645, + 151643 + ], + "pad_token_id": 151643, + "temperature": 0.6, + "top_k": 20, + "top_p": 0.95, + "transformers_version": "5.2.0" +} diff --git a/model.safetensors b/model.safetensors new file mode 100644 index 0000000..b476640 --- /dev/null +++ b/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:95bacf21dfe9f54063f1304cbace1030b135858e35f2478310f0c87a17c3467d +size 4063515640 diff --git a/qwen3_1.7b_coding_fft_20260429_153648.log b/qwen3_1.7b_coding_fft_20260429_153648.log new file mode 100644 index 0000000..5fa5f43 --- /dev/null +++ b/qwen3_1.7b_coding_fft_20260429_153648.log @@ -0,0 +1,219 @@ +==== STARTING EXPERIMENT: qwen3_1.7b_coding_fft ==== +Log File: saves/qwen3_1.7b/coding/fft/qwen3_1.7b_coding_fft_20260429_153648.log +HF Hub: https://huggingface.co/KKHYA/qwen3-1.7b-fft-coding +Timestamp: 2026-04-29 15:36:48 +===================================== +[INFO|2026-04-29 15:37:02] llamafactory.launcher:144 >> Initializing 8 distributed tasks at: 127.0.0.1:42841 +W0429 15:37:04.319000 212778 site-packages/torch/distributed/run.py:852] +W0429 15:37:04.319000 212778 site-packages/torch/distributed/run.py:852] ***************************************** +W0429 15:37:04.319000 212778 site-packages/torch/distributed/run.py:852] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +W0429 15:37:04.319000 212778 site-packages/torch/distributed/run.py:852] ***************************************** +warmup_ratio is deprecated and will be removed in v5.2. Use `warmup_steps` instead. +warmup_ratio is deprecated and will be removed in v5.2. Use `warmup_steps` instead. +warmup_ratio is deprecated and will be removed in v5.2. Use `warmup_steps` instead. +df: /root/.triton/autotune: No such file or directory +warmup_ratio is deprecated and will be removed in v5.2. Use `warmup_steps` instead. +warmup_ratio is deprecated and will be removed in v5.2. Use `warmup_steps` instead. +warmup_ratio is deprecated and will be removed in v5.2. Use `warmup_steps` instead. +warmup_ratio is deprecated and will be removed in v5.2. Use `warmup_steps` instead. +warmup_ratio is deprecated and will be removed in v5.2. Use `warmup_steps` instead. +[W429 15:37:18.649985073 ProcessGroupNCCL.cpp:929] Warning: TORCH_NCCL_AVOID_RECORD_STREAMS is the default now, this environment variable is thus deprecated. (function operator()) +[W429 15:37:18.746117814 ProcessGroupNCCL.cpp:929] Warning: TORCH_NCCL_AVOID_RECORD_STREAMS is the default now, this environment variable is thus deprecated. (function operator()) +[W429 15:37:18.778768535 ProcessGroupNCCL.cpp:929] Warning: TORCH_NCCL_AVOID_RECORD_STREAMS is the default now, this environment variable is thus deprecated. (function operator()) +[W429 15:37:18.828110308 ProcessGroupNCCL.cpp:929] Warning: TORCH_NCCL_AVOID_RECORD_STREAMS is the default now, this environment variable is thus deprecated. (function operator()) +[W429 15:37:18.868414499 ProcessGroupNCCL.cpp:929] Warning: TORCH_NCCL_AVOID_RECORD_STREAMS is the default now, this environment variable is thus deprecated. (function operator()) +[W429 15:37:18.947969051 ProcessGroupNCCL.cpp:929] Warning: TORCH_NCCL_AVOID_RECORD_STREAMS is the default now, this environment variable is thus deprecated. (function operator()) +ywang29-p4d-debug-1-worker-0:212861:212861 [0] NCCL INFO ENV/Plugin: Could not find: libnccl-env.so +ywang29-p4d-debug-1-worker-0:212861:212861 [0] NCCL INFO NCCL_SOCKET_IFNAME set by environment to ^docker0,lo +ywang29-p4d-debug-1-worker-0:212861:212861 [0] NCCL INFO Bootstrap: Using eth0:10.200.170.96<0> +ywang29-p4d-debug-1-worker-0:212861:212861 [0] NCCL INFO cudaDriverVersion 13000 +ywang29-p4d-debug-1-worker-0:212862:212862 [1] NCCL INFO ENV/Plugin: Could not find: libnccl-env.so +ywang29-p4d-debug-1-worker-0:212862:212862 [1] NCCL INFO cudaDriverVersion 13000 +ywang29-p4d-debug-1-worker-0:212862:212862 [1] NCCL INFO NCCL_SOCKET_IFNAME set by environment to ^docker0,lo +ywang29-p4d-debug-1-worker-0:212862:212862 [1] NCCL INFO Bootstrap: Using eth0:10.200.170.96<0> +ywang29-p4d-debug-1-worker-0:212862:212862 [1] NCCL INFO NCCL version 2.28.9+cuda13.0 +ywang29-p4d-debug-1-worker-0:212861:212861 [0] NCCL INFO NCCL version 2.28.9+cuda13.0 +ywang29-p4d-debug-1-worker-0:212862:212862 [1] NCCL INFO Comm config Blocking set to 1 +[W429 15:37:18.174754892 ProcessGroupNCCL.cpp:929] Warning: TORCH_NCCL_AVOID_RECORD_STREAMS is the default now, this environment variable is thus deprecated. (function operator()) +ywang29-p4d-debug-1-worker-0:212863:212863 [2] NCCL INFO ENV/Plugin: Could not find: libnccl-env.so +ywang29-p4d-debug-1-worker-0:212863:212863 [2] NCCL INFO cudaDriverVersion 13000 +ywang29-p4d-debug-1-worker-0:212863:212863 [2] NCCL INFO NCCL_SOCKET_IFNAME set by environment to ^docker0,lo +ywang29-p4d-debug-1-worker-0:212863:212863 [2] NCCL INFO Bootstrap: Using eth0:10.200.170.96<0> +ywang29-p4d-debug-1-worker-0:212863:212863 [2] NCCL INFO NCCL version 2.28.9+cuda13.0 +ywang29-p4d-debug-1-worker-0:212863:212863 [2] NCCL INFO Comm config Blocking set to 1 +ywang29-p4d-debug-1-worker-0:212867:212867 [6] NCCL INFO ENV/Plugin: Could not find: libnccl-env.so +ywang29-p4d-debug-1-worker-0:212867:212867 [6] NCCL INFO cudaDriverVersion 13000 +ywang29-p4d-debug-1-worker-0:212862:212862 [1] NCCL INFO NET/Plugin: Loaded net plugin Libfabric (v10) +ywang29-p4d-debug-1-worker-0:212862:212862 [1] NCCL INFO Successfully loaded external network plugin /opt/amazon/ofi-nccl/lib/libnccl-net.so +ywang29-p4d-debug-1-worker-0:212862:212862 [1] NCCL INFO NET/OFI Initializing aws-ofi-nccl 1.17.1 +ywang29-p4d-debug-1-worker-0:212862:212862 [1] NCCL INFO NET/OFI Using Libfabric version 2.3 +ywang29-p4d-debug-1-worker-0:212862:212862 [1] NCCL INFO NET/OFI Using CUDA driver version 13000 with runtime 13000 +ywang29-p4d-debug-1-worker-0:212862:212862 [1] NCCL INFO NET/OFI Configuring AWS-specific options +ywang29-p4d-debug-1-worker-0:212862:212862 [1] NCCL INFO NET/OFI Setting provider_filter to efa +ywang29-p4d-debug-1-worker-0:212862:212862 [1] NCCL INFO NET/OFI Running on p4de.24xlarge platform, topology file /opt/amazon/ofi-nccl/share/aws-ofi-nccl/xml/p4de-24xl-topo.xml +ywang29-p4d-debug-1-worker-0:212862:212862 [1] NCCL INFO NET/OFI Internode latency set at 75.0 us +ywang29-p4d-debug-1-worker-0:212862:212862 [1] NCCL INFO NET/OFI Using transport protocol SENDRECV (platform set) +ywang29-p4d-debug-1-worker-0:212867:212867 [6] NCCL INFO NCCL_SOCKET_IFNAME set by environment to ^docker0,lo +ywang29-p4d-debug-1-worker-0:212867:212867 [6] NCCL INFO Bootstrap: Using eth0:10.200.170.96<0> +ywang29-p4d-debug-1-worker-0:212867:212867 [6] NCCL INFO NCCL version 2.28.9+cuda13.0 + +[2026-04-29 15:37:18] ywang29-p4d-debug-1-worker-0:212862:212862 [1] int nccl_net_ofi_create_plugin(nccl_net_ofi_plugin_t**):208 NCCL WARN NET/OFI Failed to initialize sendrecv protocol + +[2026-04-29 15:37:18] ywang29-p4d-debug-1-worker-0:212862:212862 [1] int nccl_net_ofi_create_plugin(nccl_net_ofi_plugin_t**):353 NCCL WARN NET/OFI aws-ofi-nccl initialization failed + +[2026-04-29 15:37:18] ywang29-p4d-debug-1-worker-0:212862:212862 [1] ncclResult_t nccl_net_ofi_init_v6(ncclDebugLogger_t):162 NCCL WARN NET/OFI Initializing plugin failed +ywang29-p4d-debug-1-worker-0:212862:212862 [1] NCCL INFO plugin/net/net_v10.cc:81 -> 2 +ywang29-p4d-debug-1-worker-0:212862:212862 [1] NCCL INFO Failed to initialize NET plugin Libfabric +ywang29-p4d-debug-1-worker-0:212867:212867 [6] NCCL INFO Comm config Blocking set to 1 +[W429 15:37:18.265962460 ProcessGroupNCCL.cpp:929] Warning: TORCH_NCCL_AVOID_RECORD_STREAMS is the default now, this environment variable is thus deprecated. (function operator()) +ywang29-p4d-debug-1-worker-0:212865:212865 [4] NCCL INFO ENV/Plugin: Could not find: libnccl-env.so +ywang29-p4d-debug-1-worker-0:212865:212865 [4] NCCL INFO cudaDriverVersion 13000 +ywang29-p4d-debug-1-worker-0:212865:212865 [4] NCCL INFO NCCL_SOCKET_IFNAME set by environment to ^docker0,lo +ywang29-p4d-debug-1-worker-0:212865:212865 [4] NCCL INFO Bootstrap: Using eth0:10.200.170.96<0> +ywang29-p4d-debug-1-worker-0:212865:212865 [4] NCCL INFO NCCL version 2.28.9+cuda13.0 +ywang29-p4d-debug-1-worker-0:212865:212865 [4] NCCL INFO Comm config Blocking set to 1 +ywang29-p4d-debug-1-worker-0:212868:212868 [7] NCCL INFO ENV/Plugin: Could not find: libnccl-env.so +ywang29-p4d-debug-1-worker-0:212868:212868 [7] NCCL INFO cudaDriverVersion 13000 +ywang29-p4d-debug-1-worker-0:212868:212868 [7] NCCL INFO NCCL_SOCKET_IFNAME set by environment to ^docker0,lo +ywang29-p4d-debug-1-worker-0:212868:212868 [7] NCCL INFO Bootstrap: Using eth0:10.200.170.96<0> +ywang29-p4d-debug-1-worker-0:212868:212868 [7] NCCL INFO NCCL version 2.28.9+cuda13.0 +ywang29-p4d-debug-1-worker-0:212868:212868 [7] NCCL INFO Comm config Blocking set to 1 +ywang29-p4d-debug-1-worker-0:212861:212861 [0] NCCL INFO Comm config Blocking set to 1 +ywang29-p4d-debug-1-worker-0:212863:212863 [2] NCCL INFO NET/Plugin: Loaded net plugin Libfabric (v10) +ywang29-p4d-debug-1-worker-0:212863:212863 [2] NCCL INFO Successfully loaded external network plugin /opt/amazon/ofi-nccl/lib/libnccl-net.so +ywang29-p4d-debug-1-worker-0:212863:212863 [2] NCCL INFO NET/OFI Initializing aws-ofi-nccl 1.17.1 +ywang29-p4d-debug-1-worker-0:212863:212863 [2] NCCL INFO NET/OFI Using Libfabric version 2.3 +ywang29-p4d-debug-1-worker-0:212863:212863 [2] NCCL INFO NET/OFI Using CUDA driver version 13000 with runtime 13000 +ywang29-p4d-debug-1-worker-0:212863:212863 [2] NCCL INFO NET/OFI Configuring AWS-specific options +ywang29-p4d-debug-1-worker-0:212863:212863 [2] NCCL INFO NET/OFI Setting provider_filter to efa +ywang29-p4d-debug-1-worker-0:212863:212863 [2] NCCL INFO NET/OFI Running on p4de.24xlarge platform, topology file /opt/amazon/ofi-nccl/share/aws-ofi-nccl/xml/p4de-24xl-topo.xml +ywang29-p4d-debug-1-worker-0:212863:212863 [2] NCCL INFO NET/OFI Internode latency set at 75.0 us +ywang29-p4d-debug-1-worker-0:212863:212863 [2] NCCL INFO NET/OFI Using transport protocol SENDRECV (platform set) +ywang29-p4d-debug-1-worker-0:212864:212864 [3] NCCL INFO ENV/Plugin: Could not find: libnccl-env.so +ywang29-p4d-debug-1-worker-0:212864:212864 [3] NCCL INFO cudaDriverVersion 13000 +ywang29-p4d-debug-1-worker-0:212864:212864 [3] NCCL INFO NCCL_SOCKET_IFNAME set by environment to ^docker0,lo + +[2026-04-29 15:37:18] ywang29-p4d-debug-1-worker-0:212863:212863 [2] int nccl_net_ofi_create_plugin(nccl_net_ofi_plugin_t**):208 NCCL WARN NET/OFI Failed to initialize sendrecv protocol +ywang29-p4d-debug-1-worker-0:212864:212864 [3] NCCL INFO Bootstrap: Using eth0:10.200.170.96<0> + +[2026-04-29 15:37:18] ywang29-p4d-debug-1-worker-0:212863:212863 [2] int nccl_net_ofi_create_plugin(nccl_net_ofi_plugin_t**):353 NCCL WARN NET/OFI aws-ofi-nccl initialization failed + +[2026-04-29 15:37:18] ywang29-p4d-debug-1-worker-0:212863:212863 [2] ncclResult_t nccl_net_ofi_init_v6(ncclDebugLogger_t):162 NCCL WARN NET/OFI Initializing plugin failed +ywang29-p4d-debug-1-worker-0:212863:212863 [2] NCCL INFO plugin/net/net_v10.cc:81 -> 2 +ywang29-p4d-debug-1-worker-0:212864:212864 [3] NCCL INFO NCCL version 2.28.9+cuda13.0 +ywang29-p4d-debug-1-worker-0:212863:212863 [2] NCCL INFO Failed to initialize NET plugin Libfabric +ywang29-p4d-debug-1-worker-0:212867:212867 [6] NCCL INFO NET/Plugin: Loaded net plugin Libfabric (v10) +ywang29-p4d-debug-1-worker-0:212867:212867 [6] NCCL INFO Successfully loaded external network plugin /opt/amazon/ofi-nccl/lib/libnccl-net.so +ywang29-p4d-debug-1-worker-0:212867:212867 [6] NCCL INFO NET/OFI Initializing aws-ofi-nccl 1.17.1 +ywang29-p4d-debug-1-worker-0:212867:212867 [6] NCCL INFO NET/OFI Using Libfabric version 2.3 +ywang29-p4d-debug-1-worker-0:212864:212864 [3] NCCL INFO Comm config Blocking set to 1 +ywang29-p4d-debug-1-worker-0:212867:212867 [6] NCCL INFO NET/OFI Using CUDA driver version 13000 with runtime 13000 +ywang29-p4d-debug-1-worker-0:212867:212867 [6] NCCL INFO NET/OFI Configuring AWS-specific options +ywang29-p4d-debug-1-worker-0:212867:212867 [6] NCCL INFO NET/OFI Setting provider_filter to efa +ywang29-p4d-debug-1-worker-0:212867:212867 [6] NCCL INFO NET/OFI Running on p4de.24xlarge platform, topology file /opt/amazon/ofi-nccl/share/aws-ofi-nccl/xml/p4de-24xl-topo.xml +ywang29-p4d-debug-1-worker-0:212867:212867 [6] NCCL INFO NET/OFI Internode latency set at 75.0 us +ywang29-p4d-debug-1-worker-0:212867:212867 [6] NCCL INFO NET/OFI Using transport protocol SENDRECV (platform set) + +[2026-04-29 15:37:18] ywang29-p4d-debug-1-worker-0:212867:212867 [6] int nccl_net_ofi_create_plugin(nccl_net_ofi_plugin_t**):208 NCCL WARN NET/OFI Failed to initialize sendrecv protocol + +[2026-04-29 15:37:18] ywang29-p4d-debug-1-worker-0:212867:212867 [6] int nccl_net_ofi_create_plugin(nccl_net_ofi_plugin_t**):353 NCCL WARN NET/OFI aws-ofi-nccl initialization failed + +[2026-04-29 15:37:18] ywang29-p4d-debug-1-worker-0:212867:212867 [6] ncclResult_t nccl_net_ofi_init_v6(ncclDebugLogger_t):162 NCCL WARN NET/OFI Initializing plugin failed +ywang29-p4d-debug-1-worker-0:212867:212867 [6] NCCL INFO plugin/net/net_v10.cc:81 -> 2 +ywang29-p4d-debug-1-worker-0:212867:212867 [6] NCCL INFO Failed to initialize NET plugin Libfabric +ywang29-p4d-debug-1-worker-0:212866:212866 [5] NCCL INFO ENV/Plugin: Could not find: libnccl-env.so +ywang29-p4d-debug-1-worker-0:212866:212866 [5] NCCL INFO cudaDriverVersion 13000 +ywang29-p4d-debug-1-worker-0:212866:212866 [5] NCCL INFO NCCL_SOCKET_IFNAME set by environment to ^docker0,lo +ywang29-p4d-debug-1-worker-0:212866:212866 [5] NCCL INFO Bootstrap: Using eth0:10.200.170.96<0> +ywang29-p4d-debug-1-worker-0:212866:212866 [5] NCCL INFO NCCL version 2.28.9+cuda13.0 +ywang29-p4d-debug-1-worker-0:212866:212866 [5] NCCL INFO Comm config Blocking set to 1 +ywang29-p4d-debug-1-worker-0:212865:212865 [4] NCCL INFO NET/Plugin: Loaded net plugin Libfabric (v10) +ywang29-p4d-debug-1-worker-0:212865:212865 [4] NCCL INFO Successfully loaded external network plugin /opt/amazon/ofi-nccl/lib/libnccl-net.so +ywang29-p4d-debug-1-worker-0:212865:212865 [4] NCCL INFO NET/OFI Initializing aws-ofi-nccl 1.17.1 +ywang29-p4d-debug-1-worker-0:212865:212865 [4] NCCL INFO NET/OFI Using Libfabric version 2.3 +ywang29-p4d-debug-1-worker-0:212865:212865 [4] NCCL INFO NET/OFI Using CUDA driver version 13000 with runtime 13000 +ywang29-p4d-debug-1-worker-0:212865:212865 [4] NCCL INFO NET/OFI Configuring AWS-specific options +ywang29-p4d-debug-1-worker-0:212865:212865 [4] NCCL INFO NET/OFI Setting provider_filter to efa +ywang29-p4d-debug-1-worker-0:212865:212865 [4] NCCL INFO NET/OFI Running on p4de.24xlarge platform, topology file /opt/amazon/ofi-nccl/share/aws-ofi-nccl/xml/p4de-24xl-topo.xml +ywang29-p4d-debug-1-worker-0:212865:212865 [4] NCCL INFO NET/OFI Internode latency set at 75.0 us +ywang29-p4d-debug-1-worker-0:212865:212865 [4] NCCL INFO NET/OFI Using transport protocol SENDRECV (platform set) + +[2026-04-29 15:37:18] ywang29-p4d-debug-1-worker-0:212865:212865 [4] int nccl_net_ofi_create_plugin(nccl_net_ofi_plugin_t**):208 NCCL WARN NET/OFI Failed to initialize sendrecv protocol + +[2026-04-29 15:37:18] ywang29-p4d-debug-1-worker-0:212865:212865 [4] int nccl_net_ofi_create_plugin(nccl_net_ofi_plugin_t**):353 NCCL WARN NET/OFI aws-ofi-nccl initialization failed + +[2026-04-29 15:37:18] ywang29-p4d-debug-1-worker-0:212865:212865 [4] ncclResult_t nccl_net_ofi_init_v6(ncclDebugLogger_t):162 NCCL WARN NET/OFI Initializing plugin failed +ywang29-p4d-debug-1-worker-0:212865:212865 [4] NCCL INFO plugin/net/net_v10.cc:81 -> 2 +ywang29-p4d-debug-1-worker-0:212865:212865 [4] NCCL INFO Failed to initialize NET plugin Libfabric +ywang29-p4d-debug-1-worker-0:212868:212868 [7] NCCL INFO NET/Plugin: Loaded net plugin Libfabric (v10) +ywang29-p4d-debug-1-worker-0:212868:212868 [7] NCCL INFO Successfully loaded external network plugin /opt/amazon/ofi-nccl/lib/libnccl-net.so +ywang29-p4d-debug-1-worker-0:212868:212868 [7] NCCL INFO NET/OFI Initializing aws-ofi-nccl 1.17.1 +ywang29-p4d-debug-1-worker-0:212868:212868 [7] NCCL INFO NET/OFI Using Libfabric version 2.3 +ywang29-p4d-debug-1-worker-0:212868:212868 [7] NCCL INFO NET/OFI Using CUDA driver version 13000 with runtime 13000 +ywang29-p4d-debug-1-worker-0:212868:212868 [7] NCCL INFO NET/OFI Configuring AWS-specific options +ywang29-p4d-debug-1-worker-0:212868:212868 [7] NCCL INFO NET/OFI Setting provider_filter to efa +ywang29-p4d-debug-1-worker-0:212868:212868 [7] NCCL INFO NET/OFI Running on p4de.24xlarge platform, topology file /opt/amazon/ofi-nccl/share/aws-ofi-nccl/xml/p4de-24xl-topo.xml +ywang29-p4d-debug-1-worker-0:212868:212868 [7] NCCL INFO NET/OFI Internode latency set at 75.0 us +ywang29-p4d-debug-1-worker-0:212868:212868 [7] NCCL INFO NET/OFI Using transport protocol SENDRECV (platform set) + +[2026-04-29 15:37:18] ywang29-p4d-debug-1-worker-0:212868:212868 [7] int nccl_net_ofi_create_plugin(nccl_net_ofi_plugin_t**):208 NCCL WARN NET/OFI Failed to initialize sendrecv protocol + +[2026-04-29 15:37:18] ywang29-p4d-debug-1-worker-0:212868:212868 [7] int nccl_net_ofi_create_plugin(nccl_net_ofi_plugin_t**):353 NCCL WARN NET/OFI aws-ofi-nccl initialization failed + +[2026-04-29 15:37:18] ywang29-p4d-debug-1-worker-0:212868:212868 [7] ncclResult_t nccl_net_ofi_init_v6(ncclDebugLogger_t):162 NCCL WARN NET/OFI Initializing plugin failed +ywang29-p4d-debug-1-worker-0:212868:212868 [7] NCCL INFO plugin/net/net_v10.cc:81 -> 2 +ywang29-p4d-debug-1-worker-0:212868:212868 [7] NCCL INFO Failed to initialize NET plugin Libfabric +ywang29-p4d-debug-1-worker-0:212861:212861 [0] NCCL INFO NET/Plugin: Loaded net plugin Libfabric (v10) +ywang29-p4d-debug-1-worker-0:212861:212861 [0] NCCL INFO Successfully loaded external network plugin /opt/amazon/ofi-nccl/lib/libnccl-net.so +ywang29-p4d-debug-1-worker-0:212861:212861 [0] NCCL INFO NET/OFI Initializing aws-ofi-nccl 1.17.1 +ywang29-p4d-debug-1-worker-0:212861:212861 [0] NCCL INFO NET/OFI Using Libfabric version 2.3 +ywang29-p4d-debug-1-worker-0:212861:212861 [0] NCCL INFO NET/OFI Using CUDA driver version 13000 with runtime 13000 +ywang29-p4d-debug-1-worker-0:212861:212861 [0] NCCL INFO NET/OFI Configuring AWS-specific options +ywang29-p4d-debug-1-worker-0:212861:212861 [0] NCCL INFO NET/OFI Setting provider_filter to efa +ywang29-p4d-debug-1-worker-0:212861:212861 [0] NCCL INFO NET/OFI Running on p4de.24xlarge platform, topology file /opt/amazon/ofi-nccl/share/aws-ofi-nccl/xml/p4de-24xl-topo.xml +ywang29-p4d-debug-1-worker-0:212861:212861 [0] NCCL INFO NET/OFI Internode latency set at 75.0 us +ywang29-p4d-debug-1-worker-0:212861:212861 [0] NCCL INFO NET/OFI Using transport protocol SENDRECV (platform set) + +[2026-04-29 15:37:19] ywang29-p4d-debug-1-worker-0:212861:212861 [0] int nccl_net_ofi_create_plugin(nccl_net_ofi_plugin_t**):208 NCCL WARN NET/OFI Failed to initialize sendrecv protocol + +[2026-04-29 15:37:19] ywang29-p4d-debug-1-worker-0:212861:212861 [0] int nccl_net_ofi_create_plugin(nccl_net_ofi_plugin_t**):353 NCCL WARN NET/OFI aws-ofi-nccl initialization failed + +[2026-04-29 15:37:19] ywang29-p4d-debug-1-worker-0:212861:212861 [0] ncclResult_t nccl_net_ofi_init_v6(ncclDebugLogger_t):162 NCCL WARN NET/OFI Initializing plugin failed +ywang29-p4d-debug-1-worker-0:212861:212861 [0] NCCL INFO plugin/net/net_v10.cc:81 -> 2 +ywang29-p4d-debug-1-worker-0:212861:212861 [0] NCCL INFO Failed to initialize NET plugin Libfabric +ywang29-p4d-debug-1-worker-0:212864:212864 [3] NCCL INFO NET/Plugin: Loaded net plugin Libfabric (v10) +ywang29-p4d-debug-1-worker-0:212864:212864 [3] NCCL INFO Successfully loaded external network plugin /opt/amazon/ofi-nccl/lib/libnccl-net.so +ywang29-p4d-debug-1-worker-0:212864:212864 [3] NCCL INFO NET/OFI Initializing aws-ofi-nccl 1.17.1 +ywang29-p4d-debug-1-worker-0:212864:212864 [3] NCCL INFO NET/OFI Using Libfabric version 2.3 +ywang29-p4d-debug-1-worker-0:212864:212864 [3] NCCL INFO NET/OFI Using CUDA driver version 13000 with runtime 13000 +ywang29-p4d-debug-1-worker-0:212864:212864 [3] NCCL INFO NET/OFI Configuring AWS-specific options +ywang29-p4d-debug-1-worker-0:212864:212864 [3] NCCL INFO NET/OFI Setting provider_filter to efa +ywang29-p4d-debug-1-worker-0:212864:212864 [3] NCCL INFO NET/OFI Running on p4de.24xlarge platform, topology file /opt/amazon/ofi-nccl/share/aws-ofi-nccl/xml/p4de-24xl-topo.xml +ywang29-p4d-debug-1-worker-0:212864:212864 [3] NCCL INFO NET/OFI Internode latency set at 75.0 us +ywang29-p4d-debug-1-worker-0:212864:212864 [3] NCCL INFO NET/OFI Using transport protocol SENDRECV (platform set) + +[2026-04-29 15:37:19] ywang29-p4d-debug-1-worker-0:212864:212864 [3] int nccl_net_ofi_create_plugin(nccl_net_ofi_plugin_t**):208 NCCL WARN NET/OFI Failed to initialize sendrecv protocol + +[2026-04-29 15:37:19] ywang29-p4d-debug-1-worker-0:212864:212864 [3] int nccl_net_ofi_create_plugin(nccl_net_ofi_plugin_t**):353 NCCL WARN NET/OFI aws-ofi-nccl initialization failed + +[2026-04-29 15:37:19] ywang29-p4d-debug-1-worker-0:212864:212864 [3] ncclResult_t nccl_net_ofi_init_v6(ncclDebugLogger_t):162 NCCL WARN NET/OFI Initializing plugin failed +ywang29-p4d-debug-1-worker-0:212864:212864 [3] NCCL INFO plugin/net/net_v10.cc:81 -> 2 +ywang29-p4d-debug-1-worker-0:212864:212864 [3] NCCL INFO Failed to initialize NET plugin Libfabric +ywang29-p4d-debug-1-worker-0:212866:212866 [5] NCCL INFO NET/Plugin: Loaded net plugin Libfabric (v10) +ywang29-p4d-debug-1-worker-0:212866:212866 [5] NCCL INFO Successfully loaded external network plugin /opt/amazon/ofi-nccl/lib/libnccl-net.so +ywang29-p4d-debug-1-worker-0:212866:212866 [5] NCCL INFO NET/OFI Initializing aws-ofi-nccl 1.17.1 +ywang29-p4d-debug-1-worker-0:212866:212866 [5] NCCL INFO NET/OFI Using Libfabric version 2.3 +ywang29-p4d-debug-1-worker-0:212866:212866 [5] NCCL INFO NET/OFI Using CUDA driver version 13000 with runtime 13000 +ywang29-p4d-debug-1-worker-0:212866:212866 [5] NCCL INFO NET/OFI Configuring AWS-specific options +ywang29-p4d-debug-1-worker-0:212866:212866 [5] NCCL INFO NET/OFI Setting provider_filter to efa +ywang29-p4d-debug-1-worker-0:212866:212866 [5] NCCL INFO NET/OFI Running on p4de.24xlarge platform, topology file /opt/amazon/ofi-nccl/share/aws-ofi-nccl/xml/p4de-24xl-topo.xml +ywang29-p4d-debug-1-worker-0:212866:212866 [5] NCCL INFO NET/OFI Internode latency set at 75.0 us +ywang29-p4d-debug-1-worker-0:212866:212866 [5] NCCL INFO NET/OFI Using transport protocol SENDRECV (platform set) + +[2026-04-29 15:37:19] ywang29-p4d-debug-1-worker-0:212866:212866 [5] int nccl_net_ofi_create_plugin(nccl_net_ofi_plugin_t**):208 NCCL WARN NET/OFI Failed to initialize sendrecv protocol + +[2026-04-29 15:37:19] ywang29-p4d-debug-1-worker-0:212866:212866 [5] int nccl_net_ofi_create_plugin(nccl_net_ofi_plugin_t**):353 NCCL WARN NET/OFI aws-ofi-nccl initialization failed + +[2026-04-29 15:37:19] ywang29-p4d-debug-1-worker-0:212866:212866 [5] ncclResult_t nccl_net_ofi_init_v6(ncclDebugLogger_t):162 NCCL WARN NET/OFI Initializing plugin failed +ywang29-p4d-debug-1-worker-0:212866:212866 [5] NCCL INFO plugin/net/net_v10.cc:81 -> 2 +ywang29-p4d-debug-1-worker-0:212866:212866 [5] NCCL INFO Failed to initialize NET plugin Libfabric diff --git a/qwen3_1.7b_coding_fft_20260429_153746.log b/qwen3_1.7b_coding_fft_20260429_153746.log new file mode 100644 index 0000000..60aad94 --- /dev/null +++ b/qwen3_1.7b_coding_fft_20260429_153746.log @@ -0,0 +1,1343 @@ +==== STARTING EXPERIMENT: qwen3_1.7b_coding_fft ==== +Log File: saves/qwen3_1.7b/coding/fft/qwen3_1.7b_coding_fft_20260429_153746.log +HF Hub: https://huggingface.co/KKHYA/qwen3-1.7b-fft-coding +Timestamp: 2026-04-29 15:37:46 +===================================== +[INFO|2026-04-29 15:37:53] llamafactory.launcher:144 >> Initializing 8 distributed tasks at: 127.0.0.1:52079 +W0429 15:37:54.558000 213610 site-packages/torch/distributed/run.py:852] +W0429 15:37:54.558000 213610 site-packages/torch/distributed/run.py:852] ***************************************** +W0429 15:37:54.558000 213610 site-packages/torch/distributed/run.py:852] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +W0429 15:37:54.558000 213610 site-packages/torch/distributed/run.py:852] ***************************************** +warmup_ratio is deprecated and will be removed in v5.2. Use `warmup_steps` instead. +warmup_ratio is deprecated and will be removed in v5.2. Use `warmup_steps` instead. +warmup_ratio is deprecated and will be removed in v5.2. Use `warmup_steps` instead. +warmup_ratio is deprecated and will be removed in v5.2. Use `warmup_steps` instead. +warmup_ratio is deprecated and will be removed in v5.2. Use `warmup_steps` instead. +warmup_ratio is deprecated and will be removed in v5.2. Use `warmup_steps` instead. +warmup_ratio is deprecated and will be removed in v5.2. Use `warmup_steps` instead. +warmup_ratio is deprecated and will be removed in v5.2. Use `warmup_steps` instead. +[W429 15:38:05.608586790 ProcessGroupNCCL.cpp:929] Warning: TORCH_NCCL_AVOID_RECORD_STREAMS is the default now, this environment variable is thus deprecated. (function operator()) +[W429 15:38:06.629331375 ProcessGroupNCCL.cpp:929] Warning: TORCH_NCCL_AVOID_RECORD_STREAMS is the default now, this environment variable is thus deprecated. (function operator()) +[W429 15:38:06.637811309 ProcessGroupNCCL.cpp:929] Warning: TORCH_NCCL_AVOID_RECORD_STREAMS is the default now, this environment variable is thus deprecated. (function operator()) +[W429 15:38:06.648209810 ProcessGroupNCCL.cpp:929] Warning: TORCH_NCCL_AVOID_RECORD_STREAMS is the default now, this environment variable is thus deprecated. (function operator()) +[W429 15:38:06.740148869 ProcessGroupNCCL.cpp:929] Warning: TORCH_NCCL_AVOID_RECORD_STREAMS is the default now, this environment variable is thus deprecated. (function operator()) +[W429 15:38:06.830497944 ProcessGroupNCCL.cpp:929] Warning: TORCH_NCCL_AVOID_RECORD_STREAMS is the default now, this environment variable is thus deprecated. (function operator()) +ywang29-p4d-debug-1-worker-0:213681:213681 [0] NCCL INFO ENV/Plugin: Could not find: libnccl-env.so +ywang29-p4d-debug-1-worker-0:213681:213681 [0] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth +ywang29-p4d-debug-1-worker-0:213681:213681 [0] NCCL INFO Bootstrap: Using eth0:10.200.170.96<0> +ywang29-p4d-debug-1-worker-0:213681:213681 [0] NCCL INFO cudaDriverVersion 13000 +ywang29-p4d-debug-1-worker-0:213681:213681 [0] NCCL INFO NCCL version 2.28.9+cuda13.0 +[W429 15:38:06.840813307 ProcessGroupNCCL.cpp:929] Warning: TORCH_NCCL_AVOID_RECORD_STREAMS is the default now, this environment variable is thus deprecated. (function operator()) +[W429 15:38:06.954907523 ProcessGroupNCCL.cpp:929] Warning: TORCH_NCCL_AVOID_RECORD_STREAMS is the default now, this environment variable is thus deprecated. (function operator()) +ywang29-p4d-debug-1-worker-0:213686:213686 [5] NCCL INFO ENV/Plugin: Could not find: libnccl-env.so +ywang29-p4d-debug-1-worker-0:213686:213686 [5] NCCL INFO cudaDriverVersion 13000 +ywang29-p4d-debug-1-worker-0:213686:213686 [5] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth +ywang29-p4d-debug-1-worker-0:213686:213686 [5] NCCL INFO Bootstrap: Using eth0:10.200.170.96<0> +ywang29-p4d-debug-1-worker-0:213686:213686 [5] NCCL INFO NCCL version 2.28.9+cuda13.0 +ywang29-p4d-debug-1-worker-0:213686:213686 [5] NCCL INFO Comm config Blocking set to 1 +ywang29-p4d-debug-1-worker-0:213683:213683 [2] NCCL INFO ENV/Plugin: Could not find: libnccl-env.so +ywang29-p4d-debug-1-worker-0:213683:213683 [2] NCCL INFO cudaDriverVersion 13000 +ywang29-p4d-debug-1-worker-0:213683:213683 [2] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth +ywang29-p4d-debug-1-worker-0:213683:213683 [2] NCCL INFO Bootstrap: Using eth0:10.200.170.96<0> +ywang29-p4d-debug-1-worker-0:213683:213683 [2] NCCL INFO NCCL version 2.28.9+cuda13.0 +ywang29-p4d-debug-1-worker-0:213683:213683 [2] NCCL INFO Comm config Blocking set to 1 +ywang29-p4d-debug-1-worker-0:213684:213684 [3] NCCL INFO ENV/Plugin: Could not find: libnccl-env.so +ywang29-p4d-debug-1-worker-0:213684:213684 [3] NCCL INFO cudaDriverVersion 13000 +ywang29-p4d-debug-1-worker-0:213684:213684 [3] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth +ywang29-p4d-debug-1-worker-0:213684:213684 [3] NCCL INFO Bootstrap: Using eth0:10.200.170.96<0> +ywang29-p4d-debug-1-worker-0:213684:213684 [3] NCCL INFO NCCL version 2.28.9+cuda13.0 +ywang29-p4d-debug-1-worker-0:213684:213684 [3] NCCL INFO Comm config Blocking set to 1 +ywang29-p4d-debug-1-worker-0:213685:213685 [4] NCCL INFO ENV/Plugin: Could not find: libnccl-env.so +ywang29-p4d-debug-1-worker-0:213685:213685 [4] NCCL INFO cudaDriverVersion 13000 +ywang29-p4d-debug-1-worker-0:213688:213688 [7] NCCL INFO ENV/Plugin: Could not find: libnccl-env.so +ywang29-p4d-debug-1-worker-0:213688:213688 [7] NCCL INFO cudaDriverVersion 13000 +ywang29-p4d-debug-1-worker-0:213688:213688 [7] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth +ywang29-p4d-debug-1-worker-0:213688:213688 [7] NCCL INFO Bootstrap: Using eth0:10.200.170.96<0> +ywang29-p4d-debug-1-worker-0:213688:213688 [7] NCCL INFO NCCL version 2.28.9+cuda13.0 +ywang29-p4d-debug-1-worker-0:213685:213685 [4] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth +ywang29-p4d-debug-1-worker-0:213685:213685 [4] NCCL INFO Bootstrap: Using eth0:10.200.170.96<0> +ywang29-p4d-debug-1-worker-0:213685:213685 [4] NCCL INFO NCCL version 2.28.9+cuda13.0 +ywang29-p4d-debug-1-worker-0:213688:213688 [7] NCCL INFO Comm config Blocking set to 1 +ywang29-p4d-debug-1-worker-0:213685:213685 [4] NCCL INFO Comm config Blocking set to 1 +ywang29-p4d-debug-1-worker-0:213687:213687 [6] NCCL INFO ENV/Plugin: Could not find: libnccl-env.so +ywang29-p4d-debug-1-worker-0:213687:213687 [6] NCCL INFO cudaDriverVersion 13000 +ywang29-p4d-debug-1-worker-0:213687:213687 [6] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth +ywang29-p4d-debug-1-worker-0:213681:213681 [0] NCCL INFO Comm config Blocking set to 1 +ywang29-p4d-debug-1-worker-0:213687:213687 [6] NCCL INFO Bootstrap: Using eth0:10.200.170.96<0> +ywang29-p4d-debug-1-worker-0:213687:213687 [6] NCCL INFO NCCL version 2.28.9+cuda13.0 +ywang29-p4d-debug-1-worker-0:213687:213687 [6] NCCL INFO Comm config Blocking set to 1 +ywang29-p4d-debug-1-worker-0:213682:213682 [1] NCCL INFO ENV/Plugin: Could not find: libnccl-env.so +ywang29-p4d-debug-1-worker-0:213682:213682 [1] NCCL INFO cudaDriverVersion 13000 +ywang29-p4d-debug-1-worker-0:213682:213682 [1] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth +ywang29-p4d-debug-1-worker-0:213682:213682 [1] NCCL INFO Bootstrap: Using eth0:10.200.170.96<0> +ywang29-p4d-debug-1-worker-0:213682:213682 [1] NCCL INFO NCCL version 2.28.9+cuda13.0 +ywang29-p4d-debug-1-worker-0:213682:213682 [1] NCCL INFO Comm config Blocking set to 1 +ywang29-p4d-debug-1-worker-0:213686:213686 [5] NCCL INFO NET/Plugin: Loaded net plugin Libfabric (v10) +ywang29-p4d-debug-1-worker-0:213686:213686 [5] NCCL INFO Successfully loaded external network plugin /opt/amazon/ofi-nccl/lib/libnccl-net.so +ywang29-p4d-debug-1-worker-0:213686:213686 [5] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth +ywang29-p4d-debug-1-worker-0:213686:213686 [5] NCCL INFO NET/Socket : Using [0]eth0:10.200.170.96<0> +ywang29-p4d-debug-1-worker-0:213686:213686 [5] NCCL INFO Initialized NET plugin Socket +ywang29-p4d-debug-1-worker-0:213686:213686 [5] NCCL INFO Assigned NET plugin Socket to comm +ywang29-p4d-debug-1-worker-0:213686:213686 [5] NCCL INFO Using network Socket +ywang29-p4d-debug-1-worker-0:213686:213686 [5] NCCL INFO ncclCommInitRankConfig comm 0x55cadade89e0 rank 5 nranks 8 cudaDev 5 nvmlDev 5 busId 901d0 commId 0xc429a426e310f46d - Init START +ywang29-p4d-debug-1-worker-0:213683:213683 [2] NCCL INFO NET/Plugin: Loaded net plugin Libfabric (v10) +ywang29-p4d-debug-1-worker-0:213683:213683 [2] NCCL INFO Successfully loaded external network plugin /opt/amazon/ofi-nccl/lib/libnccl-net.so +ywang29-p4d-debug-1-worker-0:213683:213683 [2] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth +ywang29-p4d-debug-1-worker-0:213683:213683 [2] NCCL INFO NET/Socket : Using [0]eth0:10.200.170.96<0> +ywang29-p4d-debug-1-worker-0:213683:213683 [2] NCCL INFO Initialized NET plugin Socket +ywang29-p4d-debug-1-worker-0:213683:213683 [2] NCCL INFO Assigned NET plugin Socket to comm +ywang29-p4d-debug-1-worker-0:213683:213683 [2] NCCL INFO Using network Socket +ywang29-p4d-debug-1-worker-0:213683:213683 [2] NCCL INFO ncclCommInitRankConfig comm 0x55ec9c119990 rank 2 nranks 8 cudaDev 2 nvmlDev 2 busId 201c0 commId 0xc429a426e310f46d - Init START +ywang29-p4d-debug-1-worker-0:213684:213684 [3] NCCL INFO NET/Plugin: Loaded net plugin Libfabric (v10) +ywang29-p4d-debug-1-worker-0:213684:213684 [3] NCCL INFO Successfully loaded external network plugin /opt/amazon/ofi-nccl/lib/libnccl-net.so +ywang29-p4d-debug-1-worker-0:213684:213684 [3] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth +ywang29-p4d-debug-1-worker-0:213684:213684 [3] NCCL INFO NET/Socket : Using [0]eth0:10.200.170.96<0> +ywang29-p4d-debug-1-worker-0:213684:213684 [3] NCCL INFO Initialized NET plugin Socket +ywang29-p4d-debug-1-worker-0:213684:213684 [3] NCCL INFO Assigned NET plugin Socket to comm +ywang29-p4d-debug-1-worker-0:213684:213684 [3] NCCL INFO Using network Socket +ywang29-p4d-debug-1-worker-0:213688:213688 [7] NCCL INFO NET/Plugin: Loaded net plugin Libfabric (v10) +ywang29-p4d-debug-1-worker-0:213688:213688 [7] NCCL INFO Successfully loaded external network plugin /opt/amazon/ofi-nccl/lib/libnccl-net.so +ywang29-p4d-debug-1-worker-0:213688:213688 [7] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth +ywang29-p4d-debug-1-worker-0:213688:213688 [7] NCCL INFO NET/Socket : Using [0]eth0:10.200.170.96<0> +ywang29-p4d-debug-1-worker-0:213688:213688 [7] NCCL INFO Initialized NET plugin Socket +ywang29-p4d-debug-1-worker-0:213688:213688 [7] NCCL INFO Assigned NET plugin Socket to comm +ywang29-p4d-debug-1-worker-0:213688:213688 [7] NCCL INFO Using network Socket +ywang29-p4d-debug-1-worker-0:213684:213684 [3] NCCL INFO ncclCommInitRankConfig comm 0x5616b5c85c30 rank 3 nranks 8 cudaDev 3 nvmlDev 3 busId 201d0 commId 0xc429a426e310f46d - Init START +ywang29-p4d-debug-1-worker-0:213681:213681 [0] NCCL INFO NET/Plugin: Loaded net plugin Libfabric (v10) +ywang29-p4d-debug-1-worker-0:213681:213681 [0] NCCL INFO Successfully loaded external network plugin /opt/amazon/ofi-nccl/lib/libnccl-net.so +ywang29-p4d-debug-1-worker-0:213681:213681 [0] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth +ywang29-p4d-debug-1-worker-0:213681:213681 [0] NCCL INFO NET/Socket : Using [0]eth0:10.200.170.96<0> +ywang29-p4d-debug-1-worker-0:213681:213681 [0] NCCL INFO Initialized NET plugin Socket +ywang29-p4d-debug-1-worker-0:213681:213681 [0] NCCL INFO Assigned NET plugin Socket to comm +ywang29-p4d-debug-1-worker-0:213681:213681 [0] NCCL INFO Using network Socket +ywang29-p4d-debug-1-worker-0:213688:213688 [7] NCCL INFO ncclCommInitRankConfig comm 0x55e5dac15dd0 rank 7 nranks 8 cudaDev 7 nvmlDev 7 busId a01d0 commId 0xc429a426e310f46d - Init START +ywang29-p4d-debug-1-worker-0:213687:213687 [6] NCCL INFO NET/Plugin: Loaded net plugin Libfabric (v10) +ywang29-p4d-debug-1-worker-0:213687:213687 [6] NCCL INFO Successfully loaded external network plugin /opt/amazon/ofi-nccl/lib/libnccl-net.so +ywang29-p4d-debug-1-worker-0:213687:213687 [6] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth +ywang29-p4d-debug-1-worker-0:213687:213687 [6] NCCL INFO NET/Socket : Using [0]eth0:10.200.170.96<0> +ywang29-p4d-debug-1-worker-0:213687:213687 [6] NCCL INFO Initialized NET plugin Socket +ywang29-p4d-debug-1-worker-0:213687:213687 [6] NCCL INFO Assigned NET plugin Socket to comm +ywang29-p4d-debug-1-worker-0:213687:213687 [6] NCCL INFO Using network Socket +ywang29-p4d-debug-1-worker-0:213685:213685 [4] NCCL INFO NET/Plugin: Loaded net plugin Libfabric (v10) +ywang29-p4d-debug-1-worker-0:213685:213685 [4] NCCL INFO Successfully loaded external network plugin /opt/amazon/ofi-nccl/lib/libnccl-net.so +ywang29-p4d-debug-1-worker-0:213685:213685 [4] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth +ywang29-p4d-debug-1-worker-0:213685:213685 [4] NCCL INFO NET/Socket : Using [0]eth0:10.200.170.96<0> +ywang29-p4d-debug-1-worker-0:213685:213685 [4] NCCL INFO Initialized NET plugin Socket +ywang29-p4d-debug-1-worker-0:213685:213685 [4] NCCL INFO Assigned NET plugin Socket to comm +ywang29-p4d-debug-1-worker-0:213685:213685 [4] NCCL INFO Using network Socket +ywang29-p4d-debug-1-worker-0:213681:213681 [0] NCCL INFO ncclCommInitRankConfig comm 0x5635e92c0280 rank 0 nranks 8 cudaDev 0 nvmlDev 0 busId 101c0 commId 0xc429a426e310f46d - Init START +ywang29-p4d-debug-1-worker-0:213687:213687 [6] NCCL INFO ncclCommInitRankConfig comm 0x55e8675b14b0 rank 6 nranks 8 cudaDev 6 nvmlDev 6 busId a01c0 commId 0xc429a426e310f46d - Init START +ywang29-p4d-debug-1-worker-0:213687:213687 [6] NCCL INFO RAS client listening socket at ::1<28028> +ywang29-p4d-debug-1-worker-0:213688:213688 [7] NCCL INFO RAS client listening socket at ::1<28028> +ywang29-p4d-debug-1-worker-0:213685:213685 [4] NCCL INFO ncclCommInitRankConfig comm 0x55cfd20e4910 rank 4 nranks 8 cudaDev 4 nvmlDev 4 busId 901c0 commId 0xc429a426e310f46d - Init START +ywang29-p4d-debug-1-worker-0:213684:213684 [3] NCCL INFO RAS client listening socket at ::1<28028> +ywang29-p4d-debug-1-worker-0:213685:213685 [4] NCCL INFO RAS client listening socket at ::1<28028> +ywang29-p4d-debug-1-worker-0:213686:213686 [5] NCCL INFO RAS client listening socket at ::1<28028> +ywang29-p4d-debug-1-worker-0:213682:213682 [1] NCCL INFO NET/Plugin: Loaded net plugin Libfabric (v10) +ywang29-p4d-debug-1-worker-0:213682:213682 [1] NCCL INFO Successfully loaded external network plugin /opt/amazon/ofi-nccl/lib/libnccl-net.so +ywang29-p4d-debug-1-worker-0:213682:213682 [1] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth +ywang29-p4d-debug-1-worker-0:213682:213682 [1] NCCL INFO NET/Socket : Using [0]eth0:10.200.170.96<0> +ywang29-p4d-debug-1-worker-0:213682:213682 [1] NCCL INFO Initialized NET plugin Socket +ywang29-p4d-debug-1-worker-0:213682:213682 [1] NCCL INFO Assigned NET plugin Socket to comm +ywang29-p4d-debug-1-worker-0:213682:213682 [1] NCCL INFO Using network Socket +ywang29-p4d-debug-1-worker-0:213682:213682 [1] NCCL INFO ncclCommInitRankConfig comm 0x55730064a610 rank 1 nranks 8 cudaDev 1 nvmlDev 1 busId 101d0 commId 0xc429a426e310f46d - Init START +ywang29-p4d-debug-1-worker-0:213682:213682 [1] NCCL INFO RAS client listening socket at ::1<28028> +ywang29-p4d-debug-1-worker-0:213681:213681 [0] NCCL INFO RAS client listening socket at ::1<28028> +ywang29-p4d-debug-1-worker-0:213683:213683 [2] NCCL INFO RAS client listening socket at ::1<28028> +ywang29-p4d-debug-1-worker-0:213682:213682 [1] NCCL INFO Bootstrap timings total 0.001004 (create 0.000039, send 0.000071, recv 0.000267, ring 0.000237, delay 0.000001) +ywang29-p4d-debug-1-worker-0:213681:213681 [0] NCCL INFO Bootstrap timings total 0.042826 (create 0.000049, send 0.000085, recv 0.041999, ring 0.000241, delay 0.000001) +ywang29-p4d-debug-1-worker-0:213683:213683 [2] NCCL INFO Bootstrap timings total 0.125366 (create 0.000051, send 0.000084, recv 0.029081, ring 0.000139, delay 0.000001) +ywang29-p4d-debug-1-worker-0:213688:213688 [7] NCCL INFO Bootstrap timings total 0.083669 (create 0.000045, send 0.000077, recv 0.041036, ring 0.027744, delay 0.000001) +ywang29-p4d-debug-1-worker-0:213687:213687 [6] NCCL INFO Bootstrap timings total 0.028635 (create 0.000045, send 0.000066, recv 0.000283, ring 0.027747, delay 0.000002) +ywang29-p4d-debug-1-worker-0:213685:213685 [4] NCCL INFO Bootstrap timings total 0.023624 (create 0.000037, send 0.000067, recv 0.000181, ring 0.022940, delay 0.000001) +ywang29-p4d-debug-1-worker-0:213684:213684 [3] NCCL INFO Bootstrap timings total 0.096532 (create 0.000055, send 0.000091, recv 0.072971, ring 0.022947, delay 0.000001) +ywang29-p4d-debug-1-worker-0:213686:213686 [5] NCCL INFO Bootstrap timings total 0.167438 (create 0.000064, send 0.000211, recv 0.138854, ring 0.022860, delay 0.000001) +ywang29-p4d-debug-1-worker-0:213682:213682 [1] NCCL INFO ncclTopoGetCpuAffinity: Affinity for GPU 1 is 0-23,48-71. (GPU affinity = 0-23,48-71 ; CPU affinity = 0-95). +ywang29-p4d-debug-1-worker-0:213682:213682 [1] NCCL INFO NVLS multicast support is not available on dev 1 (NVLS_NCHANNELS 0) +ywang29-p4d-debug-1-worker-0:213683:213683 [2] NCCL INFO ncclTopoGetCpuAffinity: Affinity for GPU 2 is 0-23,48-71. (GPU affinity = 0-23,48-71 ; CPU affinity = 0-95). +ywang29-p4d-debug-1-worker-0:213683:213683 [2] NCCL INFO NVLS multicast support is not available on dev 2 (NVLS_NCHANNELS 0) +ywang29-p4d-debug-1-worker-0:213685:213685 [4] NCCL INFO ncclTopoGetCpuAffinity: Affinity for GPU 4 is 24-47,72-95. (GPU affinity = 24-47,72-95 ; CPU affinity = 0-95). +ywang29-p4d-debug-1-worker-0:213685:213685 [4] NCCL INFO NVLS multicast support is not available on dev 4 (NVLS_NCHANNELS 0) +ywang29-p4d-debug-1-worker-0:213681:213681 [0] NCCL INFO ncclTopoGetCpuAffinity: Affinity for GPU 0 is 0-23,48-71. (GPU affinity = 0-23,48-71 ; CPU affinity = 0-95). +ywang29-p4d-debug-1-worker-0:213681:213681 [0] NCCL INFO NVLS multicast support is not available on dev 0 (NVLS_NCHANNELS 0) +ywang29-p4d-debug-1-worker-0:213688:213688 [7] NCCL INFO ncclTopoGetCpuAffinity: Affinity for GPU 7 is 24-47,72-95. (GPU affinity = 24-47,72-95 ; CPU affinity = 0-95). +ywang29-p4d-debug-1-worker-0:213688:213688 [7] NCCL INFO NVLS multicast support is not available on dev 7 (NVLS_NCHANNELS 0) +ywang29-p4d-debug-1-worker-0:213686:213686 [5] NCCL INFO ncclTopoGetCpuAffinity: Affinity for GPU 5 is 24-47,72-95. (GPU affinity = 24-47,72-95 ; CPU affinity = 0-95). +ywang29-p4d-debug-1-worker-0:213686:213686 [5] NCCL INFO NVLS multicast support is not available on dev 5 (NVLS_NCHANNELS 0) +ywang29-p4d-debug-1-worker-0:213687:213687 [6] NCCL INFO ncclTopoGetCpuAffinity: Affinity for GPU 6 is 24-47,72-95. (GPU affinity = 24-47,72-95 ; CPU affinity = 0-95). +ywang29-p4d-debug-1-worker-0:213687:213687 [6] NCCL INFO NVLS multicast support is not available on dev 6 (NVLS_NCHANNELS 0) +ywang29-p4d-debug-1-worker-0:213684:213684 [3] NCCL INFO ncclTopoGetCpuAffinity: Affinity for GPU 3 is 0-23,48-71. (GPU affinity = 0-23,48-71 ; CPU affinity = 0-95). +ywang29-p4d-debug-1-worker-0:213684:213684 [3] NCCL INFO NVLS multicast support is not available on dev 3 (NVLS_NCHANNELS 0) +ywang29-p4d-debug-1-worker-0:213682:213682 [1] NCCL INFO comm 0x55730064a610 rank 1 nRanks 8 nNodes 1 localRanks 8 localRank 1 MNNVL 0 +ywang29-p4d-debug-1-worker-0:213681:213681 [0] NCCL INFO comm 0x5635e92c0280 rank 0 nRanks 8 nNodes 1 localRanks 8 localRank 0 MNNVL 0 +ywang29-p4d-debug-1-worker-0:213688:213688 [7] NCCL INFO comm 0x55e5dac15dd0 rank 7 nRanks 8 nNodes 1 localRanks 8 localRank 7 MNNVL 0 +ywang29-p4d-debug-1-worker-0:213687:213687 [6] NCCL INFO comm 0x55e8675b14b0 rank 6 nRanks 8 nNodes 1 localRanks 8 localRank 6 MNNVL 0 +ywang29-p4d-debug-1-worker-0:213686:213686 [5] NCCL INFO comm 0x55cadade89e0 rank 5 nRanks 8 nNodes 1 localRanks 8 localRank 5 MNNVL 0 +ywang29-p4d-debug-1-worker-0:213683:213683 [2] NCCL INFO comm 0x55ec9c119990 rank 2 nRanks 8 nNodes 1 localRanks 8 localRank 2 MNNVL 0 +ywang29-p4d-debug-1-worker-0:213684:213684 [3] NCCL INFO comm 0x5616b5c85c30 rank 3 nRanks 8 nNodes 1 localRanks 8 localRank 3 MNNVL 0 +ywang29-p4d-debug-1-worker-0:213685:213685 [4] NCCL INFO comm 0x55cfd20e4910 rank 4 nRanks 8 nNodes 1 localRanks 8 localRank 4 MNNVL 0 +ywang29-p4d-debug-1-worker-0:213681:213681 [0] NCCL INFO Channel 00/24 : 0 1 2 3 4 5 6 7 +ywang29-p4d-debug-1-worker-0:213681:213681 [0] NCCL INFO Channel 01/24 : 0 1 2 3 4 5 6 7 +ywang29-p4d-debug-1-worker-0:213681:213681 [0] NCCL INFO Channel 02/24 : 0 1 2 3 4 5 6 7 +ywang29-p4d-debug-1-worker-0:213681:213681 [0] NCCL INFO Channel 03/24 : 0 1 2 3 4 5 6 7 +ywang29-p4d-debug-1-worker-0:213681:213681 [0] NCCL INFO Channel 04/24 : 0 1 2 3 4 5 6 7 +ywang29-p4d-debug-1-worker-0:213681:213681 [0] NCCL INFO Channel 05/24 : 0 1 2 3 4 5 6 7 +ywang29-p4d-debug-1-worker-0:213681:213681 [0] NCCL INFO Channel 06/24 : 0 1 2 3 4 5 6 7 +ywang29-p4d-debug-1-worker-0:213681:213681 [0] NCCL INFO Channel 07/24 : 0 1 2 3 4 5 6 7 +ywang29-p4d-debug-1-worker-0:213682:213682 [1] NCCL INFO Trees [0] 2/-1/-1->1->0 [1] 2/-1/-1->1->0 [2] 2/-1/-1->1->0 [3] 2/-1/-1->1->0 [4] 2/-1/-1->1->0 [5] 2/-1/-1->1->0 [6] 2/-1/-1->1->0 [7] 2/-1/-1->1->0 [8] 2/-1/-1->1->0 [9] 2/-1/-1->1->0 [10] 2/-1/-1->1->0 [11] 2/-1/-1->1->0 [12] 2/-1/-1->1->0 [13] 2/-1/-1->1->0 [14] 2/-1/-1->1->0 [15] 2/-1/-1->1->0 [16] 2/-1/-1->1->0 [17] 2/-1/-1->1->0 [18] 2/-1/-1->1->0 [19] 2/-1/-1->1->0 [20] 2/-1/-1->1->0 [21] 2/-1/-1->1->0 [22] 2/-1/-1->1->0 [23] 2/-1/-1->1->0 +ywang29-p4d-debug-1-worker-0:213681:213681 [0] NCCL INFO Channel 08/24 : 0 1 2 3 4 5 6 7 +ywang29-p4d-debug-1-worker-0:213688:213688 [7] NCCL INFO Trees [0] -1/-1/-1->7->6 [1] -1/-1/-1->7->6 [2] -1/-1/-1->7->6 [3] -1/-1/-1->7->6 [4] -1/-1/-1->7->6 [5] -1/-1/-1->7->6 [6] -1/-1/-1->7->6 [7] -1/-1/-1->7->6 [8] -1/-1/-1->7->6 [9] -1/-1/-1->7->6 [10] -1/-1/-1->7->6 [11] -1/-1/-1->7->6 [12] -1/-1/-1->7->6 [13] -1/-1/-1->7->6 [14] -1/-1/-1->7->6 [15] -1/-1/-1->7->6 [16] -1/-1/-1->7->6 [17] -1/-1/-1->7->6 [18] -1/-1/-1->7->6 [19] -1/-1/-1->7->6 [20] -1/-1/-1->7->6 [21] -1/-1/-1->7->6 [22] -1/-1/-1->7->6 [23] -1/-1/-1->7->6 +ywang29-p4d-debug-1-worker-0:213687:213687 [6] NCCL INFO Trees [0] 7/-1/-1->6->5 [1] 7/-1/-1->6->5 [2] 7/-1/-1->6->5 [3] 7/-1/-1->6->5 [4] 7/-1/-1->6->5 [5] 7/-1/-1->6->5 [6] 7/-1/-1->6->5 [7] 7/-1/-1->6->5 [8] 7/-1/-1->6->5 [9] 7/-1/-1->6->5 [10] 7/-1/-1->6->5 [11] 7/-1/-1->6->5 [12] 7/-1/-1->6->5 [13] 7/-1/-1->6->5 [14] 7/-1/-1->6->5 [15] 7/-1/-1->6->5 [16] 7/-1/-1->6->5 [17] 7/-1/-1->6->5 [18] 7/-1/-1->6->5 [19] 7/-1/-1->6->5 [20] 7/-1/-1->6->5 [21] 7/-1/-1->6->5 [22] 7/-1/-1->6->5 [23] 7/-1/-1->6->5 +ywang29-p4d-debug-1-worker-0:213686:213686 [5] NCCL INFO Trees [0] 6/-1/-1->5->4 [1] 6/-1/-1->5->4 [2] 6/-1/-1->5->4 [3] 6/-1/-1->5->4 [4] 6/-1/-1->5->4 [5] 6/-1/-1->5->4 [6] 6/-1/-1->5->4 [7] 6/-1/-1->5->4 [8] 6/-1/-1->5->4 [9] 6/-1/-1->5->4 [10] 6/-1/-1->5->4 [11] 6/-1/-1->5->4 [12] 6/-1/-1->5->4 [13] 6/-1/-1->5->4 [14] 6/-1/-1->5->4 [15] 6/-1/-1->5->4 [16] 6/-1/-1->5->4 [17] 6/-1/-1->5->4 [18] 6/-1/-1->5->4 [19] 6/-1/-1->5->4 [20] 6/-1/-1->5->4 [21] 6/-1/-1->5->4 [22] 6/-1/-1->5->4 [23] 6/-1/-1->5->4 +ywang29-p4d-debug-1-worker-0:213681:213681 [0] NCCL INFO Channel 09/24 : 0 1 2 3 4 5 6 7 +ywang29-p4d-debug-1-worker-0:213684:213684 [3] NCCL INFO Trees [0] 4/-1/-1->3->2 [1] 4/-1/-1->3->2 [2] 4/-1/-1->3->2 [3] 4/-1/-1->3->2 [4] 4/-1/-1->3->2 [5] 4/-1/-1->3->2 [6] 4/-1/-1->3->2 [7] 4/-1/-1->3->2 [8] 4/-1/-1->3->2 [9] 4/-1/-1->3->2 [10] 4/-1/-1->3->2 [11] 4/-1/-1->3->2 [12] 4/-1/-1->3->2 [13] 4/-1/-1->3->2 [14] 4/-1/-1->3->2 [15] 4/-1/-1->3->2 [16] 4/-1/-1->3->2 [17] 4/-1/-1->3->2 [18] 4/-1/-1->3->2 [19] 4/-1/-1->3->2 [20] 4/-1/-1->3->2 [21] 4/-1/-1->3->2 [22] 4/-1/-1->3->2 [23] 4/-1/-1->3->2 +ywang29-p4d-debug-1-worker-0:213682:213682 [1] NCCL INFO P2P Chunksize set to 524288 +ywang29-p4d-debug-1-worker-0:213683:213683 [2] NCCL INFO Trees [0] 3/-1/-1->2->1 [1] 3/-1/-1->2->1 [2] 3/-1/-1->2->1 [3] 3/-1/-1->2->1 [4] 3/-1/-1->2->1 [5] 3/-1/-1->2->1 [6] 3/-1/-1->2->1 [7] 3/-1/-1->2->1 [8] 3/-1/-1->2->1 [9] 3/-1/-1->2->1 [10] 3/-1/-1->2->1 [11] 3/-1/-1->2->1 [12] 3/-1/-1->2->1 [13] 3/-1/-1->2->1 [14] 3/-1/-1->2->1 [15] 3/-1/-1->2->1 [16] 3/-1/-1->2->1 [17] 3/-1/-1->2->1 [18] 3/-1/-1->2->1 [19] 3/-1/-1->2->1 [20] 3/-1/-1->2->1 [21] 3/-1/-1->2->1 [22] 3/-1/-1->2->1 [23] 3/-1/-1->2->1 +ywang29-p4d-debug-1-worker-0:213681:213681 [0] NCCL INFO Channel 10/24 : 0 1 2 3 4 5 6 7 +ywang29-p4d-debug-1-worker-0:213685:213685 [4] NCCL INFO Trees [0] 5/-1/-1->4->3 [1] 5/-1/-1->4->3 [2] 5/-1/-1->4->3 [3] 5/-1/-1->4->3 [4] 5/-1/-1->4->3 [5] 5/-1/-1->4->3 [6] 5/-1/-1->4->3 [7] 5/-1/-1->4->3 [8] 5/-1/-1->4->3 [9] 5/-1/-1->4->3 [10] 5/-1/-1->4->3 [11] 5/-1/-1->4->3 [12] 5/-1/-1->4->3 [13] 5/-1/-1->4->3 [14] 5/-1/-1->4->3 [15] 5/-1/-1->4->3 [16] 5/-1/-1->4->3 [17] 5/-1/-1->4->3 [18] 5/-1/-1->4->3 [19] 5/-1/-1->4->3 [20] 5/-1/-1->4->3 [21] 5/-1/-1->4->3 [22] 5/-1/-1->4->3 [23] 5/-1/-1->4->3 +ywang29-p4d-debug-1-worker-0:213687:213687 [6] NCCL INFO P2P Chunksize set to 524288 +ywang29-p4d-debug-1-worker-0:213686:213686 [5] NCCL INFO P2P Chunksize set to 524288 +ywang29-p4d-debug-1-worker-0:213688:213688 [7] NCCL INFO P2P Chunksize set to 524288 +ywang29-p4d-debug-1-worker-0:213681:213681 [0] NCCL INFO Channel 11/24 : 0 1 2 3 4 5 6 7 +ywang29-p4d-debug-1-worker-0:213684:213684 [3] NCCL INFO P2P Chunksize set to 524288 +ywang29-p4d-debug-1-worker-0:213683:213683 [2] NCCL INFO P2P Chunksize set to 524288 +ywang29-p4d-debug-1-worker-0:213681:213681 [0] NCCL INFO Channel 12/24 : 0 1 2 3 4 5 6 7 +ywang29-p4d-debug-1-worker-0:213685:213685 [4] NCCL INFO P2P Chunksize set to 524288 +ywang29-p4d-debug-1-worker-0:213681:213681 [0] NCCL INFO Channel 13/24 : 0 1 2 3 4 5 6 7 +ywang29-p4d-debug-1-worker-0:213681:213681 [0] NCCL INFO Channel 14/24 : 0 1 2 3 4 5 6 7 +ywang29-p4d-debug-1-worker-0:213681:213681 [0] NCCL INFO Channel 15/24 : 0 1 2 3 4 5 6 7 +ywang29-p4d-debug-1-worker-0:213681:213681 [0] NCCL INFO Channel 16/24 : 0 1 2 3 4 5 6 7 +ywang29-p4d-debug-1-worker-0:213681:213681 [0] NCCL INFO Channel 17/24 : 0 1 2 3 4 5 6 7 +ywang29-p4d-debug-1-worker-0:213681:213681 [0] NCCL INFO Channel 18/24 : 0 1 2 3 4 5 6 7 +ywang29-p4d-debug-1-worker-0:213681:213681 [0] NCCL INFO Channel 19/24 : 0 1 2 3 4 5 6 7 +ywang29-p4d-debug-1-worker-0:213681:213681 [0] NCCL INFO Channel 20/24 : 0 1 2 3 4 5 6 7 +ywang29-p4d-debug-1-worker-0:213681:213681 [0] NCCL INFO Channel 21/24 : 0 1 2 3 4 5 6 7 +ywang29-p4d-debug-1-worker-0:213681:213681 [0] NCCL INFO Channel 22/24 : 0 1 2 3 4 5 6 7 +ywang29-p4d-debug-1-worker-0:213681:213681 [0] NCCL INFO Channel 23/24 : 0 1 2 3 4 5 6 7 +ywang29-p4d-debug-1-worker-0:213681:213681 [0] NCCL INFO Trees [0] 1/-1/-1->0->-1 [1] 1/-1/-1->0->-1 [2] 1/-1/-1->0->-1 [3] 1/-1/-1->0->-1 [4] 1/-1/-1->0->-1 [5] 1/-1/-1->0->-1 [6] 1/-1/-1->0->-1 [7] 1/-1/-1->0->-1 [8] 1/-1/-1->0->-1 [9] 1/-1/-1->0->-1 [10] 1/-1/-1->0->-1 [11] 1/-1/-1->0->-1 [12] 1/-1/-1->0->-1 [13] 1/-1/-1->0->-1 [14] 1/-1/-1->0->-1 [15] 1/-1/-1->0->-1 [16] 1/-1/-1->0->-1 [17] 1/-1/-1->0->-1 [18] 1/-1/-1->0->-1 [19] 1/-1/-1->0->-1 [20] 1/-1/-1->0->-1 [21] 1/-1/-1->0->-1 [22] 1/-1/-1->0->-1 [23] 1/-1/-1->0->-1 +ywang29-p4d-debug-1-worker-0:213681:213681 [0] NCCL INFO P2P Chunksize set to 524288 +ywang29-p4d-debug-1-worker-0:213684:213684 [3] NCCL INFO PROFILER/Plugin: Could not find: libnccl-profiler.so +ywang29-p4d-debug-1-worker-0:213684:213684 [3] NCCL INFO Check P2P Type isAllDirectP2p 1 directMode 0 isAllCudaP2p 1 +ywang29-p4d-debug-1-worker-0:213684:214061 [0] NCCL INFO [Proxy Service UDS] Device 3 CPU core 6 +ywang29-p4d-debug-1-worker-0:213684:214060 [0] NCCL INFO [Proxy Service] Device 3 CPU core 50 +ywang29-p4d-debug-1-worker-0:213686:213686 [5] NCCL INFO PROFILER/Plugin: Could not find: libnccl-profiler.so +ywang29-p4d-debug-1-worker-0:213686:213686 [5] NCCL INFO Check P2P Type isAllDirectP2p 1 directMode 0 isAllCudaP2p 1 +ywang29-p4d-debug-1-worker-0:213688:213688 [7] NCCL INFO PROFILER/Plugin: Could not find: libnccl-profiler.so +ywang29-p4d-debug-1-worker-0:213688:213688 [7] NCCL INFO Check P2P Type isAllDirectP2p 1 directMode 0 isAllCudaP2p 1 +ywang29-p4d-debug-1-worker-0:213686:214062 [0] NCCL INFO [Proxy Service] Device 5 CPU core 27 +ywang29-p4d-debug-1-worker-0:213686:214063 [0] NCCL INFO [Proxy Service UDS] Device 5 CPU core 30 +ywang29-p4d-debug-1-worker-0:213688:214064 [0] NCCL INFO [Proxy Service] Device 7 CPU core 83 +ywang29-p4d-debug-1-worker-0:213688:214065 [0] NCCL INFO [Proxy Service UDS] Device 7 CPU core 33 +ywang29-p4d-debug-1-worker-0:213682:213682 [1] NCCL INFO PROFILER/Plugin: Could not find: libnccl-profiler.so +ywang29-p4d-debug-1-worker-0:213682:213682 [1] NCCL INFO Check P2P Type isAllDirectP2p 1 directMode 0 isAllCudaP2p 1 +ywang29-p4d-debug-1-worker-0:213682:214067 [0] NCCL INFO [Proxy Service UDS] Device 1 CPU core 3 +ywang29-p4d-debug-1-worker-0:213682:214066 [0] NCCL INFO [Proxy Service] Device 1 CPU core 64 +ywang29-p4d-debug-1-worker-0:213681:213681 [0] NCCL INFO PROFILER/Plugin: Could not find: libnccl-profiler.so +ywang29-p4d-debug-1-worker-0:213681:213681 [0] NCCL INFO Check P2P Type isAllDirectP2p 1 directMode 0 isAllCudaP2p 1 +ywang29-p4d-debug-1-worker-0:213681:214068 [0] NCCL INFO [Proxy Service] Device 0 CPU core 63 +ywang29-p4d-debug-1-worker-0:213681:214069 [0] NCCL INFO [Proxy Service UDS] Device 0 CPU core 4 +ywang29-p4d-debug-1-worker-0:213685:213685 [4] NCCL INFO PROFILER/Plugin: Could not find: libnccl-profiler.so +ywang29-p4d-debug-1-worker-0:213685:213685 [4] NCCL INFO Check P2P Type isAllDirectP2p 1 directMode 0 isAllCudaP2p 1 +ywang29-p4d-debug-1-worker-0:213683:213683 [2] NCCL INFO PROFILER/Plugin: Could not find: libnccl-profiler.so +ywang29-p4d-debug-1-worker-0:213683:213683 [2] NCCL INFO Check P2P Type isAllDirectP2p 1 directMode 0 isAllCudaP2p 1 +ywang29-p4d-debug-1-worker-0:213685:214070 [0] NCCL INFO [Proxy Service] Device 4 CPU core 89 +ywang29-p4d-debug-1-worker-0:213687:213687 [6] NCCL INFO PROFILER/Plugin: Could not find: libnccl-profiler.so +ywang29-p4d-debug-1-worker-0:213685:214071 [0] NCCL INFO [Proxy Service UDS] Device 4 CPU core 32 +ywang29-p4d-debug-1-worker-0:213687:213687 [6] NCCL INFO Check P2P Type isAllDirectP2p 1 directMode 0 isAllCudaP2p 1 +ywang29-p4d-debug-1-worker-0:213683:214072 [0] NCCL INFO [Proxy Service] Device 2 CPU core 12 +ywang29-p4d-debug-1-worker-0:213683:214073 [0] NCCL INFO [Proxy Service UDS] Device 2 CPU core 14 +ywang29-p4d-debug-1-worker-0:213687:214074 [0] NCCL INFO [Proxy Service] Device 6 CPU core 87 +ywang29-p4d-debug-1-worker-0:213687:214075 [0] NCCL INFO [Proxy Service UDS] Device 6 CPU core 95 +ywang29-p4d-debug-1-worker-0:213682:213682 [1] NCCL INFO TUNER/Plugin: Could not find: libnccl-tuner.so +ywang29-p4d-debug-1-worker-0:213682:213682 [1] NCCL INFO TUNER/Plugin: Using nccl_ofi_tuner (v3) +ywang29-p4d-debug-1-worker-0:213682:213682 [1] NCCL INFO NET/OFI NCCL_OFI_TUNER is not available for platform : p4de.24xlarge, Fall back to NCCL's tuner +ywang29-p4d-debug-1-worker-0:213682:213682 [1] NCCL INFO threadThresholds 8/8/64 | 64/8/64 | 512 | 512 +ywang29-p4d-debug-1-worker-0:213682:213682 [1] NCCL INFO 24 coll channels, 24 collnet channels, 0 nvls channels, 32 p2p channels, 32 p2p channels per peer +ywang29-p4d-debug-1-worker-0:213686:213686 [5] NCCL INFO TUNER/Plugin: Could not find: libnccl-tuner.so +ywang29-p4d-debug-1-worker-0:213686:213686 [5] NCCL INFO TUNER/Plugin: Using nccl_ofi_tuner (v3) +ywang29-p4d-debug-1-worker-0:213686:213686 [5] NCCL INFO NET/OFI NCCL_OFI_TUNER is not available for platform : p4de.24xlarge, Fall back to NCCL's tuner +ywang29-p4d-debug-1-worker-0:213686:213686 [5] NCCL INFO threadThresholds 8/8/64 | 64/8/64 | 512 | 512 +ywang29-p4d-debug-1-worker-0:213686:213686 [5] NCCL INFO 24 coll channels, 24 collnet channels, 0 nvls channels, 32 p2p channels, 32 p2p channels per peer +ywang29-p4d-debug-1-worker-0:213684:213684 [3] NCCL INFO TUNER/Plugin: Could not find: libnccl-tuner.so +ywang29-p4d-debug-1-worker-0:213684:213684 [3] NCCL INFO TUNER/Plugin: Using nccl_ofi_tuner (v3) +ywang29-p4d-debug-1-worker-0:213684:213684 [3] NCCL INFO NET/OFI NCCL_OFI_TUNER is not available for platform : p4de.24xlarge, Fall back to NCCL's tuner +ywang29-p4d-debug-1-worker-0:213684:213684 [3] NCCL INFO threadThresholds 8/8/64 | 64/8/64 | 512 | 512 +ywang29-p4d-debug-1-worker-0:213684:213684 [3] NCCL INFO 24 coll channels, 24 collnet channels, 0 nvls channels, 32 p2p channels, 32 p2p channels per peer +ywang29-p4d-debug-1-worker-0:213681:213681 [0] NCCL INFO TUNER/Plugin: Could not find: libnccl-tuner.so +ywang29-p4d-debug-1-worker-0:213681:213681 [0] NCCL INFO TUNER/Plugin: Using nccl_ofi_tuner (v3) +ywang29-p4d-debug-1-worker-0:213681:213681 [0] NCCL INFO NET/OFI NCCL_OFI_TUNER is not available for platform : p4de.24xlarge, Fall back to NCCL's tuner +ywang29-p4d-debug-1-worker-0:213681:213681 [0] NCCL INFO threadThresholds 8/8/64 | 64/8/64 | 512 | 512 +ywang29-p4d-debug-1-worker-0:213681:213681 [0] NCCL INFO 24 coll channels, 24 collnet channels, 0 nvls channels, 32 p2p channels, 32 p2p channels per peer +ywang29-p4d-debug-1-worker-0:213681:213681 [0] NCCL INFO CC Off, workFifoBytes 1048576 +ywang29-p4d-debug-1-worker-0:213687:213687 [6] NCCL INFO TUNER/Plugin: Could not find: libnccl-tuner.so +ywang29-p4d-debug-1-worker-0:213687:213687 [6] NCCL INFO TUNER/Plugin: Using nccl_ofi_tuner (v3) +ywang29-p4d-debug-1-worker-0:213687:213687 [6] NCCL INFO NET/OFI NCCL_OFI_TUNER is not available for platform : p4de.24xlarge, Fall back to NCCL's tuner +ywang29-p4d-debug-1-worker-0:213687:213687 [6] NCCL INFO threadThresholds 8/8/64 | 64/8/64 | 512 | 512 +ywang29-p4d-debug-1-worker-0:213687:213687 [6] NCCL INFO 24 coll channels, 24 collnet channels, 0 nvls channels, 32 p2p channels, 32 p2p channels per peer +ywang29-p4d-debug-1-worker-0:213685:213685 [4] NCCL INFO TUNER/Plugin: Could not find: libnccl-tuner.so +ywang29-p4d-debug-1-worker-0:213685:213685 [4] NCCL INFO TUNER/Plugin: Using nccl_ofi_tuner (v3) +ywang29-p4d-debug-1-worker-0:213685:213685 [4] NCCL INFO NET/OFI NCCL_OFI_TUNER is not available for platform : p4de.24xlarge, Fall back to NCCL's tuner +ywang29-p4d-debug-1-worker-0:213685:213685 [4] NCCL INFO threadThresholds 8/8/64 | 64/8/64 | 512 | 512 +ywang29-p4d-debug-1-worker-0:213685:213685 [4] NCCL INFO 24 coll channels, 24 collnet channels, 0 nvls channels, 32 p2p channels, 32 p2p channels per peer +ywang29-p4d-debug-1-worker-0:213688:213688 [7] NCCL INFO TUNER/Plugin: Could not find: libnccl-tuner.so +ywang29-p4d-debug-1-worker-0:213688:213688 [7] NCCL INFO TUNER/Plugin: Using nccl_ofi_tuner (v3) +ywang29-p4d-debug-1-worker-0:213688:213688 [7] NCCL INFO NET/OFI NCCL_OFI_TUNER is not available for platform : p4de.24xlarge, Fall back to NCCL's tuner +ywang29-p4d-debug-1-worker-0:213688:213688 [7] NCCL INFO threadThresholds 8/8/64 | 64/8/64 | 512 | 512 +ywang29-p4d-debug-1-worker-0:213688:213688 [7] NCCL INFO 24 coll channels, 24 collnet channels, 0 nvls channels, 32 p2p channels, 32 p2p channels per peer +ywang29-p4d-debug-1-worker-0:213683:213683 [2] NCCL INFO TUNER/Plugin: Could not find: libnccl-tuner.so +ywang29-p4d-debug-1-worker-0:213683:213683 [2] NCCL INFO TUNER/Plugin: Using nccl_ofi_tuner (v3) +ywang29-p4d-debug-1-worker-0:213683:213683 [2] NCCL INFO NET/OFI NCCL_OFI_TUNER is not available for platform : p4de.24xlarge, Fall back to NCCL's tuner +ywang29-p4d-debug-1-worker-0:213683:213683 [2] NCCL INFO threadThresholds 8/8/64 | 64/8/64 | 512 | 512 +ywang29-p4d-debug-1-worker-0:213683:213683 [2] NCCL INFO 24 coll channels, 24 collnet channels, 0 nvls channels, 32 p2p channels, 32 p2p channels per peer +ywang29-p4d-debug-1-worker-0:213688:213688 [7] NCCL INFO ncclCommInitRankConfig comm 0x55e5dac15dd0 rank 7 nranks 8 cudaDev 7 nvmlDev 7 busId a01d0 commId 0xc429a426e310f46d - Init COMPLETE +ywang29-p4d-debug-1-worker-0:213684:213684 [3] NCCL INFO ncclCommInitRankConfig comm 0x5616b5c85c30 rank 3 nranks 8 cudaDev 3 nvmlDev 3 busId 201d0 commId 0xc429a426e310f46d - Init COMPLETE +ywang29-p4d-debug-1-worker-0:213686:213686 [5] NCCL INFO ncclCommInitRankConfig comm 0x55cadade89e0 rank 5 nranks 8 cudaDev 5 nvmlDev 5 busId 901d0 commId 0xc429a426e310f46d - Init COMPLETE +ywang29-p4d-debug-1-worker-0:213681:213681 [0] NCCL INFO ncclCommInitRankConfig comm 0x5635e92c0280 rank 0 nranks 8 cudaDev 0 nvmlDev 0 busId 101c0 commId 0xc429a426e310f46d - Init COMPLETE +ywang29-p4d-debug-1-worker-0:213685:213685 [4] NCCL INFO ncclCommInitRankConfig comm 0x55cfd20e4910 rank 4 nranks 8 cudaDev 4 nvmlDev 4 busId 901c0 commId 0xc429a426e310f46d - Init COMPLETE +ywang29-p4d-debug-1-worker-0:213682:213682 [1] NCCL INFO ncclCommInitRankConfig comm 0x55730064a610 rank 1 nranks 8 cudaDev 1 nvmlDev 1 busId 101d0 commId 0xc429a426e310f46d - Init COMPLETE +ywang29-p4d-debug-1-worker-0:213684:213684 [3] NCCL INFO Init timings - ncclCommInitRankConfig: rank 3 nranks 8 total 0.44 (kernels 0.18, alloc 0.03, bootstrap 0.10, allgathers 0.00, topo 0.04, graphs 0.00, connections 0.06, rest 0.02) +ywang29-p4d-debug-1-worker-0:213688:213688 [7] NCCL INFO Init timings - ncclCommInitRankConfig: rank 7 nranks 8 total 0.43 (kernels 0.19, alloc 0.03, bootstrap 0.08, allgathers 0.01, topo 0.04, graphs 0.00, connections 0.06, rest 0.03) +ywang29-p4d-debug-1-worker-0:213686:213686 [5] NCCL INFO Init timings - ncclCommInitRankConfig: rank 5 nranks 8 total 0.50 (kernels 0.18, alloc 0.03, bootstrap 0.17, allgathers 0.01, topo 0.04, graphs 0.00, connections 0.06, rest 0.03) +ywang29-p4d-debug-1-worker-0:213681:213681 [0] NCCL INFO Init timings - ncclCommInitRankConfig: rank 0 nranks 8 total 0.40 (kernels 0.18, alloc 0.05, bootstrap 0.04, allgathers 0.01, topo 0.04, graphs 0.00, connections 0.05, rest 0.03) +ywang29-p4d-debug-1-worker-0:213685:213685 [4] NCCL INFO Init timings - ncclCommInitRankConfig: rank 4 nranks 8 total 0.43 (kernels 0.23, alloc 0.05, bootstrap 0.02, allgathers 0.01, topo 0.04, graphs 0.00, connections 0.06, rest 0.03) +ywang29-p4d-debug-1-worker-0:213682:213682 [1] NCCL INFO Init timings - ncclCommInitRankConfig: rank 1 nranks 8 total 0.38 (kernels 0.23, alloc 0.02, bootstrap 0.00, allgathers 0.01, topo 0.03, graphs 0.00, connections 0.04, rest 0.05) +ywang29-p4d-debug-1-worker-0:213687:213687 [6] NCCL INFO ncclCommInitRankConfig comm 0x55e8675b14b0 rank 6 nranks 8 cudaDev 6 nvmlDev 6 busId a01c0 commId 0xc429a426e310f46d - Init COMPLETE +ywang29-p4d-debug-1-worker-0:213683:213683 [2] NCCL INFO ncclCommInitRankConfig comm 0x55ec9c119990 rank 2 nranks 8 cudaDev 2 nvmlDev 2 busId 201c0 commId 0xc429a426e310f46d - Init COMPLETE +ywang29-p4d-debug-1-worker-0:213687:213687 [6] NCCL INFO Init timings - ncclCommInitRankConfig: rank 6 nranks 8 total 0.40 (kernels 0.19, alloc 0.05, bootstrap 0.03, allgathers 0.01, topo 0.04, graphs 0.00, connections 0.06, rest 0.03) +ywang29-p4d-debug-1-worker-0:213683:213683 [2] NCCL INFO Init timings - ncclCommInitRankConfig: rank 2 nranks 8 total 0.46 (kernels 0.18, alloc 0.02, bootstrap 0.13, allgathers 0.01, topo 0.03, graphs 0.00, connections 0.06, rest 0.03) +[INFO|2026-04-29 15:38:07] llamafactory.hparams.parser:505 >> Process rank: 1, world size: 8, device: cuda:1, distributed training: True, compute dtype: torch.bfloat16 +[INFO|2026-04-29 15:38:07] llamafactory.hparams.parser:505 >> Process rank: 7, world size: 8, device: cuda:7, distributed training: True, compute dtype: torch.bfloat16 +[INFO|2026-04-29 15:38:07] llamafactory.hparams.parser:505 >> Process rank: 2, world size: 8, device: cuda:2, distributed training: True, compute dtype: torch.bfloat16 +[INFO|2026-04-29 15:38:07] llamafactory.hparams.parser:505 >> Process rank: 5, world size: 8, device: cuda:5, distributed training: True, compute dtype: torch.bfloat16 +[INFO|2026-04-29 15:38:07] llamafactory.hparams.parser:505 >> Process rank: 4, world size: 8, device: cuda:4, distributed training: True, compute dtype: torch.bfloat16 +[INFO|2026-04-29 15:38:07] llamafactory.hparams.parser:505 >> Process rank: 3, world size: 8, device: cuda:3, distributed training: True, compute dtype: torch.bfloat16 +[INFO|2026-04-29 15:38:07] llamafactory.hparams.parser:505 >> Process rank: 6, world size: 8, device: cuda:6, distributed training: True, compute dtype: torch.bfloat16 +[INFO|2026-04-29 15:38:07] llamafactory.hparams.parser:505 >> Process rank: 0, world size: 8, device: cuda:0, distributed training: True, compute dtype: torch.bfloat16 +[INFO|configuration_utils.py:670] 2026-04-29 15:38:07,681 >> loading configuration file config.json from cache at /root/.cache/huggingface/hub/models--Qwen--Qwen3-1.7B/snapshots/70d244cc86ccca08cf5af4e1e306ecf908b1ad5e/config.json +[INFO|configuration_utils.py:742] 2026-04-29 15:38:07,684 >> Model config Qwen3Config { + "architectures": [ + "Qwen3ForCausalLM" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "bos_token_id": 151643, + "dtype": "bfloat16", + "eos_token_id": 151645, + "head_dim": 128, + "hidden_act": "silu", + "hidden_size": 2048, + "initializer_range": 0.02, + "intermediate_size": 6144, + "layer_types": [ + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention" + ], + "masked_layers": null, + "max_position_embeddings": 40960, + "max_window_layers": 28, + "model_type": "qwen3", + "num_attention_heads": 16, + "num_hidden_layers": 28, + "num_key_value_heads": 8, + "pad_token_id": null, + "rms_norm_eps": 1e-06, + "rope_parameters": { + "rope_theta": 1000000, + "rope_type": "default" + }, + "sliding_window": null, + "sparsity_attn": null, + "sparsity_mlp": null, + "subnet_mode": null, + "subnet_type": null, + "threshold_attn": null, + "threshold_mlp": null, + "tie_word_embeddings": true, + "transformers_version": "5.2.0", + "use_cache": true, + "use_sliding_window": false, + "vocab_size": 151936 +} + +[INFO|configuration_utils.py:670] 2026-04-29 15:38:11,634 >> loading configuration file config.json from cache at /root/.cache/huggingface/hub/models--Qwen--Qwen3-1.7B/snapshots/70d244cc86ccca08cf5af4e1e306ecf908b1ad5e/config.json +[INFO|configuration_utils.py:742] 2026-04-29 15:38:11,635 >> Model config Qwen3Config { + "architectures": [ + "Qwen3ForCausalLM" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "bos_token_id": 151643, + "dtype": "bfloat16", + "eos_token_id": 151645, + "head_dim": 128, + "hidden_act": "silu", + "hidden_size": 2048, + "initializer_range": 0.02, + "intermediate_size": 6144, + "layer_types": [ + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention" + ], + "masked_layers": null, + "max_position_embeddings": 40960, + "max_window_layers": 28, + "model_type": "qwen3", + "num_attention_heads": 16, + "num_hidden_layers": 28, + "num_key_value_heads": 8, + "pad_token_id": null, + "rms_norm_eps": 1e-06, + "rope_parameters": { + "rope_theta": 1000000, + "rope_type": "default" + }, + "sliding_window": null, + "sparsity_attn": null, + "sparsity_mlp": null, + "subnet_mode": null, + "subnet_type": null, + "threshold_attn": null, + "threshold_mlp": null, + "tie_word_embeddings": true, + "transformers_version": "5.2.0", + "use_cache": true, + "use_sliding_window": false, + "vocab_size": 151936 +} + +[INFO|configuration_utils.py:670] 2026-04-29 15:38:11,758 >> loading configuration file config.json from cache at /root/.cache/huggingface/hub/models--Qwen--Qwen3-1.7B/snapshots/70d244cc86ccca08cf5af4e1e306ecf908b1ad5e/config.json +[INFO|configuration_utils.py:742] 2026-04-29 15:38:11,759 >> Model config Qwen3Config { + "architectures": [ + "Qwen3ForCausalLM" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "bos_token_id": 151643, + "dtype": "bfloat16", + "eos_token_id": 151645, + "head_dim": 128, + "hidden_act": "silu", + "hidden_size": 2048, + "initializer_range": 0.02, + "intermediate_size": 6144, + "layer_types": [ + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention" + ], + "masked_layers": null, + "max_position_embeddings": 40960, + "max_window_layers": 28, + "model_type": "qwen3", + "num_attention_heads": 16, + "num_hidden_layers": 28, + "num_key_value_heads": 8, + "pad_token_id": null, + "rms_norm_eps": 1e-06, + "rope_parameters": { + "rope_theta": 1000000, + "rope_type": "default" + }, + "sliding_window": null, + "sparsity_attn": null, + "sparsity_mlp": null, + "subnet_mode": null, + "subnet_type": null, + "threshold_attn": null, + "threshold_mlp": null, + "tie_word_embeddings": true, + "transformers_version": "5.2.0", + "use_cache": true, + "use_sliding_window": false, + "vocab_size": 151936 +} + +ywang29-p4d-debug-1-worker-0:213686:213686 [5] NCCL INFO Channel 00/0 : 5[5] -> 6[6] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213686:213686 [5] NCCL INFO Channel 01/0 : 5[5] -> 6[6] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213686:213686 [5] NCCL INFO Channel 02/0 : 5[5] -> 6[6] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213686:213686 [5] NCCL INFO Channel 03/0 : 5[5] -> 6[6] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213686:213686 [5] NCCL INFO Channel 04/0 : 5[5] -> 6[6] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213686:213686 [5] NCCL INFO Channel 05/0 : 5[5] -> 6[6] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213686:213686 [5] NCCL INFO Channel 06/0 : 5[5] -> 6[6] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213686:213686 [5] NCCL INFO Channel 07/0 : 5[5] -> 6[6] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213686:213686 [5] NCCL INFO Channel 08/0 : 5[5] -> 6[6] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213686:213686 [5] NCCL INFO Channel 09/0 : 5[5] -> 6[6] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213686:213686 [5] NCCL INFO Channel 10/0 : 5[5] -> 6[6] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213686:213686 [5] NCCL INFO Channel 11/0 : 5[5] -> 6[6] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213686:213686 [5] NCCL INFO Channel 12/0 : 5[5] -> 6[6] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213686:213686 [5] NCCL INFO Channel 13/0 : 5[5] -> 6[6] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213686:213686 [5] NCCL INFO Channel 14/0 : 5[5] -> 6[6] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213686:213686 [5] NCCL INFO Channel 15/0 : 5[5] -> 6[6] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213686:213686 [5] NCCL INFO Channel 16/0 : 5[5] -> 6[6] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213686:213686 [5] NCCL INFO Channel 17/0 : 5[5] -> 6[6] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213686:213686 [5] NCCL INFO Channel 18/0 : 5[5] -> 6[6] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213686:213686 [5] NCCL INFO Channel 19/0 : 5[5] -> 6[6] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213686:213686 [5] NCCL INFO Channel 20/0 : 5[5] -> 6[6] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213686:213686 [5] NCCL INFO Channel 21/0 : 5[5] -> 6[6] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213686:213686 [5] NCCL INFO Channel 22/0 : 5[5] -> 6[6] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213686:213686 [5] NCCL INFO Channel 23/0 : 5[5] -> 6[6] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213684:213684 [3] NCCL INFO Channel 00/0 : 3[3] -> 4[4] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213684:213684 [3] NCCL INFO Channel 01/0 : 3[3] -> 4[4] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213684:213684 [3] NCCL INFO Channel 02/0 : 3[3] -> 4[4] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213684:213684 [3] NCCL INFO Channel 03/0 : 3[3] -> 4[4] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213684:213684 [3] NCCL INFO Channel 04/0 : 3[3] -> 4[4] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213684:213684 [3] NCCL INFO Channel 05/0 : 3[3] -> 4[4] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213684:213684 [3] NCCL INFO Channel 06/0 : 3[3] -> 4[4] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213684:213684 [3] NCCL INFO Channel 07/0 : 3[3] -> 4[4] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213684:213684 [3] NCCL INFO Channel 08/0 : 3[3] -> 4[4] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213684:213684 [3] NCCL INFO Channel 09/0 : 3[3] -> 4[4] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213684:213684 [3] NCCL INFO Channel 10/0 : 3[3] -> 4[4] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213684:213684 [3] NCCL INFO Channel 11/0 : 3[3] -> 4[4] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213684:213684 [3] NCCL INFO Channel 12/0 : 3[3] -> 4[4] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213684:213684 [3] NCCL INFO Channel 13/0 : 3[3] -> 4[4] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213684:213684 [3] NCCL INFO Channel 14/0 : 3[3] -> 4[4] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213684:213684 [3] NCCL INFO Channel 15/0 : 3[3] -> 4[4] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213684:213684 [3] NCCL INFO Channel 16/0 : 3[3] -> 4[4] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213684:213684 [3] NCCL INFO Channel 17/0 : 3[3] -> 4[4] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213684:213684 [3] NCCL INFO Channel 18/0 : 3[3] -> 4[4] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213684:213684 [3] NCCL INFO Channel 19/0 : 3[3] -> 4[4] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213684:213684 [3] NCCL INFO Channel 20/0 : 3[3] -> 4[4] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213684:213684 [3] NCCL INFO Channel 21/0 : 3[3] -> 4[4] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213684:213684 [3] NCCL INFO Channel 22/0 : 3[3] -> 4[4] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213684:213684 [3] NCCL INFO Channel 23/0 : 3[3] -> 4[4] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213688:213688 [7] NCCL INFO Channel 00/0 : 7[7] -> 0[0] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213688:213688 [7] NCCL INFO Channel 01/0 : 7[7] -> 0[0] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213688:213688 [7] NCCL INFO Channel 02/0 : 7[7] -> 0[0] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213688:213688 [7] NCCL INFO Channel 03/0 : 7[7] -> 0[0] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213688:213688 [7] NCCL INFO Channel 04/0 : 7[7] -> 0[0] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213688:213688 [7] NCCL INFO Channel 05/0 : 7[7] -> 0[0] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213688:213688 [7] NCCL INFO Channel 06/0 : 7[7] -> 0[0] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213688:213688 [7] NCCL INFO Channel 07/0 : 7[7] -> 0[0] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213688:213688 [7] NCCL INFO Channel 08/0 : 7[7] -> 0[0] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213688:213688 [7] NCCL INFO Channel 09/0 : 7[7] -> 0[0] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213688:213688 [7] NCCL INFO Channel 10/0 : 7[7] -> 0[0] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213688:213688 [7] NCCL INFO Channel 11/0 : 7[7] -> 0[0] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213688:213688 [7] NCCL INFO Channel 12/0 : 7[7] -> 0[0] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213688:213688 [7] NCCL INFO Channel 13/0 : 7[7] -> 0[0] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213688:213688 [7] NCCL INFO Channel 14/0 : 7[7] -> 0[0] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213688:213688 [7] NCCL INFO Channel 15/0 : 7[7] -> 0[0] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213688:213688 [7] NCCL INFO Channel 16/0 : 7[7] -> 0[0] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213688:213688 [7] NCCL INFO Channel 17/0 : 7[7] -> 0[0] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213688:213688 [7] NCCL INFO Channel 18/0 : 7[7] -> 0[0] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213688:213688 [7] NCCL INFO Channel 19/0 : 7[7] -> 0[0] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213688:213688 [7] NCCL INFO Channel 20/0 : 7[7] -> 0[0] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213688:213688 [7] NCCL INFO Channel 21/0 : 7[7] -> 0[0] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213688:213688 [7] NCCL INFO Channel 22/0 : 7[7] -> 0[0] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213688:213688 [7] NCCL INFO Channel 23/0 : 7[7] -> 0[0] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213682:213682 [1] NCCL INFO Channel 00/0 : 1[1] -> 2[2] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213682:213682 [1] NCCL INFO Channel 01/0 : 1[1] -> 2[2] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213682:213682 [1] NCCL INFO Channel 02/0 : 1[1] -> 2[2] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213682:213682 [1] NCCL INFO Channel 03/0 : 1[1] -> 2[2] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213682:213682 [1] NCCL INFO Channel 04/0 : 1[1] -> 2[2] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213682:213682 [1] NCCL INFO Channel 05/0 : 1[1] -> 2[2] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213682:213682 [1] NCCL INFO Channel 06/0 : 1[1] -> 2[2] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213682:213682 [1] NCCL INFO Channel 07/0 : 1[1] -> 2[2] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213682:213682 [1] NCCL INFO Channel 08/0 : 1[1] -> 2[2] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213682:213682 [1] NCCL INFO Channel 09/0 : 1[1] -> 2[2] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213682:213682 [1] NCCL INFO Channel 10/0 : 1[1] -> 2[2] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213682:213682 [1] NCCL INFO Channel 11/0 : 1[1] -> 2[2] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213682:213682 [1] NCCL INFO Channel 12/0 : 1[1] -> 2[2] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213682:213682 [1] NCCL INFO Channel 13/0 : 1[1] -> 2[2] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213682:213682 [1] NCCL INFO Channel 14/0 : 1[1] -> 2[2] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213682:213682 [1] NCCL INFO Channel 15/0 : 1[1] -> 2[2] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213682:213682 [1] NCCL INFO Channel 16/0 : 1[1] -> 2[2] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213682:213682 [1] NCCL INFO Channel 17/0 : 1[1] -> 2[2] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213682:213682 [1] NCCL INFO Channel 18/0 : 1[1] -> 2[2] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213682:213682 [1] NCCL INFO Channel 19/0 : 1[1] -> 2[2] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213682:213682 [1] NCCL INFO Channel 20/0 : 1[1] -> 2[2] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213682:213682 [1] NCCL INFO Channel 21/0 : 1[1] -> 2[2] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213682:213682 [1] NCCL INFO Channel 22/0 : 1[1] -> 2[2] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213682:213682 [1] NCCL INFO Channel 23/0 : 1[1] -> 2[2] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213685:213685 [4] NCCL INFO Channel 00/0 : 4[4] -> 5[5] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213685:213685 [4] NCCL INFO Channel 01/0 : 4[4] -> 5[5] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213685:213685 [4] NCCL INFO Channel 02/0 : 4[4] -> 5[5] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213685:213685 [4] NCCL INFO Channel 03/0 : 4[4] -> 5[5] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213685:213685 [4] NCCL INFO Channel 04/0 : 4[4] -> 5[5] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213685:213685 [4] NCCL INFO Channel 05/0 : 4[4] -> 5[5] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213685:213685 [4] NCCL INFO Channel 06/0 : 4[4] -> 5[5] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213685:213685 [4] NCCL INFO Channel 07/0 : 4[4] -> 5[5] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213685:213685 [4] NCCL INFO Channel 08/0 : 4[4] -> 5[5] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213685:213685 [4] NCCL INFO Channel 09/0 : 4[4] -> 5[5] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213685:213685 [4] NCCL INFO Channel 10/0 : 4[4] -> 5[5] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213685:213685 [4] NCCL INFO Channel 11/0 : 4[4] -> 5[5] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213685:213685 [4] NCCL INFO Channel 12/0 : 4[4] -> 5[5] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213685:213685 [4] NCCL INFO Channel 13/0 : 4[4] -> 5[5] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213685:213685 [4] NCCL INFO Channel 14/0 : 4[4] -> 5[5] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213685:213685 [4] NCCL INFO Channel 15/0 : 4[4] -> 5[5] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213685:213685 [4] NCCL INFO Channel 16/0 : 4[4] -> 5[5] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213685:213685 [4] NCCL INFO Channel 17/0 : 4[4] -> 5[5] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213685:213685 [4] NCCL INFO Channel 18/0 : 4[4] -> 5[5] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213685:213685 [4] NCCL INFO Channel 19/0 : 4[4] -> 5[5] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213685:213685 [4] NCCL INFO Channel 20/0 : 4[4] -> 5[5] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213685:213685 [4] NCCL INFO Channel 21/0 : 4[4] -> 5[5] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213685:213685 [4] NCCL INFO Channel 22/0 : 4[4] -> 5[5] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213685:213685 [4] NCCL INFO Channel 23/0 : 4[4] -> 5[5] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213683:213683 [2] NCCL INFO Channel 00/0 : 2[2] -> 3[3] via P2P/CUMEM/read +[INFO|2026-04-29 15:38:13] llamafactory.data.loader:144 >> Loading dataset allenai/tulu-3-sft-personas-code... +ywang29-p4d-debug-1-worker-0:213683:213683 [2] NCCL INFO Channel 01/0 : 2[2] -> 3[3] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213683:213683 [2] NCCL INFO Channel 02/0 : 2[2] -> 3[3] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213683:213683 [2] NCCL INFO Channel 03/0 : 2[2] -> 3[3] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213683:213683 [2] NCCL INFO Channel 04/0 : 2[2] -> 3[3] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213683:213683 [2] NCCL INFO Channel 05/0 : 2[2] -> 3[3] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213683:213683 [2] NCCL INFO Channel 06/0 : 2[2] -> 3[3] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213683:213683 [2] NCCL INFO Channel 07/0 : 2[2] -> 3[3] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213683:213683 [2] NCCL INFO Channel 08/0 : 2[2] -> 3[3] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213683:213683 [2] NCCL INFO Channel 09/0 : 2[2] -> 3[3] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213683:213683 [2] NCCL INFO Channel 10/0 : 2[2] -> 3[3] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213683:213683 [2] NCCL INFO Channel 11/0 : 2[2] -> 3[3] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213683:213683 [2] NCCL INFO Channel 12/0 : 2[2] -> 3[3] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213683:213683 [2] NCCL INFO Channel 13/0 : 2[2] -> 3[3] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213683:213683 [2] NCCL INFO Channel 14/0 : 2[2] -> 3[3] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213683:213683 [2] NCCL INFO Channel 15/0 : 2[2] -> 3[3] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213683:213683 [2] NCCL INFO Channel 16/0 : 2[2] -> 3[3] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213683:213683 [2] NCCL INFO Channel 17/0 : 2[2] -> 3[3] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213683:213683 [2] NCCL INFO Channel 18/0 : 2[2] -> 3[3] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213683:213683 [2] NCCL INFO Channel 19/0 : 2[2] -> 3[3] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213683:213683 [2] NCCL INFO Channel 20/0 : 2[2] -> 3[3] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213683:213683 [2] NCCL INFO Channel 21/0 : 2[2] -> 3[3] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213683:213683 [2] NCCL INFO Channel 22/0 : 2[2] -> 3[3] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213683:213683 [2] NCCL INFO Channel 23/0 : 2[2] -> 3[3] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213687:213687 [6] NCCL INFO Channel 00/0 : 6[6] -> 7[7] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213687:213687 [6] NCCL INFO Channel 01/0 : 6[6] -> 7[7] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213687:213687 [6] NCCL INFO Channel 02/0 : 6[6] -> 7[7] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213687:213687 [6] NCCL INFO Channel 03/0 : 6[6] -> 7[7] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213687:213687 [6] NCCL INFO Channel 04/0 : 6[6] -> 7[7] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213687:213687 [6] NCCL INFO Channel 05/0 : 6[6] -> 7[7] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213687:213687 [6] NCCL INFO Channel 06/0 : 6[6] -> 7[7] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213687:213687 [6] NCCL INFO Channel 07/0 : 6[6] -> 7[7] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213687:213687 [6] NCCL INFO Channel 08/0 : 6[6] -> 7[7] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213687:213687 [6] NCCL INFO Channel 09/0 : 6[6] -> 7[7] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213687:213687 [6] NCCL INFO Channel 10/0 : 6[6] -> 7[7] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213687:213687 [6] NCCL INFO Channel 11/0 : 6[6] -> 7[7] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213687:213687 [6] NCCL INFO Channel 12/0 : 6[6] -> 7[7] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213687:213687 [6] NCCL INFO Channel 13/0 : 6[6] -> 7[7] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213687:213687 [6] NCCL INFO Channel 14/0 : 6[6] -> 7[7] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213687:213687 [6] NCCL INFO Channel 15/0 : 6[6] -> 7[7] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213687:213687 [6] NCCL INFO Channel 16/0 : 6[6] -> 7[7] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213687:213687 [6] NCCL INFO Channel 17/0 : 6[6] -> 7[7] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213687:213687 [6] NCCL INFO Channel 18/0 : 6[6] -> 7[7] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213687:213687 [6] NCCL INFO Channel 19/0 : 6[6] -> 7[7] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213687:213687 [6] NCCL INFO Channel 20/0 : 6[6] -> 7[7] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213687:213687 [6] NCCL INFO Channel 21/0 : 6[6] -> 7[7] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213687:213687 [6] NCCL INFO Channel 22/0 : 6[6] -> 7[7] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213687:213687 [6] NCCL INFO Channel 23/0 : 6[6] -> 7[7] via P2P/CUMEM/read +Setting num_proc from 16 back to 1 for the train split to disable multiprocessing as it only contains one shard. + Generating train split: 0%| | 0/34999 [00:00> Sampled 10000 examples from dataset allenai/tulu-3-sft-personas-code. + Converting format of dataset (num_proc=16): 0%| | 0/10000 [00:00> Loading dataset KKHYA/evol_codealpaca_converted... +Repo card metadata block was not found. Setting CardData to empty. +Setting num_proc from 16 back to 1 for the train split to disable multiprocessing as it only contains one shard. + Generating train split: 0%| | 0/110999 [00:00> Sampled 10000 examples from dataset KKHYA/evol_codealpaca_converted. + Converting format of dataset (num_proc=16): 0%| | 0/10000 [00:00> Loading dataset KKHYA/codefeedback_filtered_instructions_converted... +Repo card metadata block was not found. Setting CardData to empty. +Setting num_proc from 16 to 2 for the train split as it only contains 2 shards. + Generating train split: 0%| | 0/156361 [00:00> Sampled 10000 examples from dataset KKHYA/codefeedback_filtered_instructions_converted. + Converting format of dataset (num_proc=16): 0%| | 0/10000 [00:00 1[1] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213681:213681 [0] NCCL INFO Channel 01/0 : 0[0] -> 1[1] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213681:213681 [0] NCCL INFO Channel 02/0 : 0[0] -> 1[1] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213681:213681 [0] NCCL INFO Channel 03/0 : 0[0] -> 1[1] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213681:213681 [0] NCCL INFO Channel 04/0 : 0[0] -> 1[1] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213681:213681 [0] NCCL INFO Channel 05/0 : 0[0] -> 1[1] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213681:213681 [0] NCCL INFO Channel 06/0 : 0[0] -> 1[1] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213681:213681 [0] NCCL INFO Channel 07/0 : 0[0] -> 1[1] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213681:213681 [0] NCCL INFO Channel 08/0 : 0[0] -> 1[1] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213681:213681 [0] NCCL INFO Channel 09/0 : 0[0] -> 1[1] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213681:213681 [0] NCCL INFO Channel 10/0 : 0[0] -> 1[1] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213681:213681 [0] NCCL INFO Channel 11/0 : 0[0] -> 1[1] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213681:213681 [0] NCCL INFO Channel 12/0 : 0[0] -> 1[1] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213681:213681 [0] NCCL INFO Channel 13/0 : 0[0] -> 1[1] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213681:213681 [0] NCCL INFO Channel 14/0 : 0[0] -> 1[1] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213681:213681 [0] NCCL INFO Channel 15/0 : 0[0] -> 1[1] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213681:213681 [0] NCCL INFO Channel 16/0 : 0[0] -> 1[1] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213681:213681 [0] NCCL INFO Channel 17/0 : 0[0] -> 1[1] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213681:213681 [0] NCCL INFO Channel 18/0 : 0[0] -> 1[1] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213681:213681 [0] NCCL INFO Channel 19/0 : 0[0] -> 1[1] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213681:213681 [0] NCCL INFO Channel 20/0 : 0[0] -> 1[1] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213681:213681 [0] NCCL INFO Channel 21/0 : 0[0] -> 1[1] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213681:213681 [0] NCCL INFO Channel 22/0 : 0[0] -> 1[1] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213681:213681 [0] NCCL INFO Channel 23/0 : 0[0] -> 1[1] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213682:213682 [1] NCCL INFO Connected all rings, use ring PXN 0 GDR 1 +ywang29-p4d-debug-1-worker-0:213681:213681 [0] NCCL INFO Connected all rings, use ring PXN 0 GDR 1 +ywang29-p4d-debug-1-worker-0:213683:213683 [2] NCCL INFO Connected all rings, use ring PXN 0 GDR 1 +ywang29-p4d-debug-1-worker-0:213688:213688 [7] NCCL INFO Connected all rings, use ring PXN 0 GDR 1 +ywang29-p4d-debug-1-worker-0:213687:213687 [6] NCCL INFO Connected all rings, use ring PXN 0 GDR 1 +ywang29-p4d-debug-1-worker-0:213684:213684 [3] NCCL INFO Connected all rings, use ring PXN 0 GDR 1 +ywang29-p4d-debug-1-worker-0:213685:213685 [4] NCCL INFO Connected all rings, use ring PXN 0 GDR 1 +ywang29-p4d-debug-1-worker-0:213686:213686 [5] NCCL INFO Connected all rings, use ring PXN 0 GDR 1 + Running tokenizer on dataset (num_proc=16): 0%| | 0/30000 [00:00user +Write a python function to calculate the total number of goals scored by Alanyaspor in a given season from a list of match results. Each match result is represented as a dictionary with keys "home_team", "away_team", "home_goals", and "away_goals". Alanyaspor could be either the home or away team in any match. The input is a list of such match result dictionaries, and the output should be an integer representing the total number of goals scored by Alanyaspor. + +Input: +- `match_results`: A list of dictionaries, where each dictionary contains: + - "home_team" (string): The name of the home team. + - "away_team" (string): The name of the away team. + - "home_goals" (int): The number of goals scored by the home team. + - "away_goals" (int): The number of goals scored by the away team. + +Output: +- An integer representing the total number of goals scored by Alanyaspor. + +Example: +```python +match_results = [ + {"home_team": "Alanyaspor", "away_team": "TeamA", "home_goals": 2, "away_goals": 1}, + {"home_team": "TeamB", "away_team": "Alanyaspor", "home_goals": 3, "away_goals": 2}, + {"home_team": "Alanyaspor", "away_team": "TeamC", "home_goals": 1, "away_goals": 1}, + {"home_team": "TeamD", "away_team": "Alanyaspor", "home_goals": 0, "away_goals": 3} +] +# Expected Output: 8 +```<|im_end|> +<|im_start|>assistant +def calculate_alanyaspor_goals(match_results): + total_goals = 0 + for match in match_results: + if match["home_team"] == "Alanyaspor": + total_goals += match["home_goals"] + elif match["away_team"] == "Alanyaspor": + total_goals += match["away_goals"] + return total_goals<|im_end|> + +label_ids: +[-100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, 750, 11047, 8418, 3767, 300, 4308, 96244, 25401, 13576, 982, 262, 2790, 96244, 284, 220, 15, 198, 262, 369, 2432, 304, 2432, 13576, 510, 286, 421, 2432, 1183, 5117, 26532, 1341, 621, 330, 2101, 3767, 300, 4308, 4660, 310, 2790, 96244, 1421, 2432, 1183, 5117, 96244, 7026, 286, 4409, 2432, 1183, 13757, 26532, 1341, 621, 330, 2101, 3767, 300, 4308, 4660, 310, 2790, 96244, 1421, 2432, 1183, 13757, 96244, 7026, 262, 470, 2790, 96244, 151645, 198] +labels: +def calculate_alanyaspor_goals(match_results): + total_goals = 0 + for match in match_results: + if match["home_team"] == "Alanyaspor": + total_goals += match["home_goals"] + elif match["away_team"] == "Alanyaspor": + total_goals += match["away_goals"] + return total_goals<|im_end|> + +[INFO|configuration_utils.py:670] 2026-04-29 15:39:04,827 >> loading configuration file config.json from cache at /root/.cache/huggingface/hub/models--Qwen--Qwen3-1.7B/snapshots/70d244cc86ccca08cf5af4e1e306ecf908b1ad5e/config.json +[INFO|configuration_utils.py:742] 2026-04-29 15:39:04,828 >> Model config Qwen3Config { + "architectures": [ + "Qwen3ForCausalLM" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "bos_token_id": 151643, + "dtype": "bfloat16", + "eos_token_id": 151645, + "head_dim": 128, + "hidden_act": "silu", + "hidden_size": 2048, + "initializer_range": 0.02, + "intermediate_size": 6144, + "layer_types": [ + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention" + ], + "masked_layers": null, + "max_position_embeddings": 40960, + "max_window_layers": 28, + "model_type": "qwen3", + "num_attention_heads": 16, + "num_hidden_layers": 28, + "num_key_value_heads": 8, + "pad_token_id": null, + "rms_norm_eps": 1e-06, + "rope_parameters": { + "rope_theta": 1000000, + "rope_type": "default" + }, + "sliding_window": null, + "sparsity_attn": null, + "sparsity_mlp": null, + "subnet_mode": null, + "subnet_type": null, + "threshold_attn": null, + "threshold_mlp": null, + "tie_word_embeddings": true, + "transformers_version": "5.2.0", + "use_cache": true, + "use_sliding_window": false, + "vocab_size": 151936 +} + +[INFO|2026-04-29 15:39:04] llamafactory.model.model_utils.kv_cache:144 >> KV cache is disabled during training. +[INFO|modeling_utils.py:710] 2026-04-29 15:39:06,415 >> loading weights file model.safetensors from cache at /root/.cache/huggingface/hub/models--Qwen--Qwen3-1.7B/snapshots/70d244cc86ccca08cf5af4e1e306ecf908b1ad5e/model.safetensors.index.json + Fetching 2 files: 0%| | 0/2 [00:00> Will use dtype=torch.bfloat16 as defined in model's config object +[INFO|configuration_utils.py:1014] 2026-04-29 15:39:11,368 >> Generate config GenerationConfig { + "bos_token_id": 151643, + "eos_token_id": 151645, + "output_attentions": false, + "output_hidden_states": false, + "use_cache": false +} + + Fetching 2 files: 50%|█████ | 1/2 [00:04<00:04, 4.49s/it] Fetching 2 files: 100%|██████████| 2/2 [00:04<00:00, 2.24s/it] + Fetching 2 files: 50%|█████ | 1/2 [00:04<00:04, 4.44s/it] Fetching 2 files: 100%|██████████| 2/2 [00:04<00:00, 2.22s/it] + Fetching 2 files: 50%|█████ | 1/2 [00:04<00:04, 4.53s/it] Fetching 2 files: 100%|██████████| 2/2 [00:04<00:00, 2.26s/it] + Fetching 2 files: 50%|█████ | 1/2 [00:04<00:04, 4.33s/it] Fetching 2 files: 100%|██████████| 2/2 [00:04<00:00, 2.16s/it] + Fetching 2 files: 50%|█████ | 1/2 [00:04<00:04, 4.53s/it] Fetching 2 files: 100%|██████████| 2/2 [00:04<00:00, 2.27s/it] + Fetching 2 files: 50%|█████ | 1/2 [00:04<00:04, 4.48s/it] Fetching 2 files: 100%|██████████| 2/2 [00:04<00:00, 2.24s/it] + Fetching 2 files: 50%|█████ | 1/2 [00:04<00:04, 4.48s/it] Fetching 2 files: 100%|██████████| 2/2 [00:04<00:00, 2.24s/it] + Loading weights: 0%| | 0/311 [00:00> The tied weights mapping and config for this model specifies to tie model.embed_tokens.weight to lm_head.weight, but both are present in the checkpoints, so we will NOT tie them. You should update the config with `tie_word_embeddings=False` to silence this warning +The tied weights mapping and config for this model specifies to tie model.embed_tokens.weight to lm_head.weight, but both are present in the checkpoints, so we will NOT tie them. You should update the config with `tie_word_embeddings=False` to silence this warning +The tokenizer has new PAD/BOS/EOS tokens that differ from the model config and generation config. The model config and generation config were aligned accordingly, being updated with the tokenizer's values. Updated tokens: {'bos_token_id': None, 'pad_token_id': 151643}. +The tokenizer has new PAD/BOS/EOS tokens that differ from the model config and generation config. The model config and generation config were aligned accordingly, being updated with the tokenizer's values. Updated tokens: {'bos_token_id': None, 'pad_token_id': 151643}. +[INFO|configuration_utils.py:967] 2026-04-29 15:39:13,129 >> loading configuration file generation_config.json from cache at /root/.cache/huggingface/hub/models--Qwen--Qwen3-1.7B/snapshots/70d244cc86ccca08cf5af4e1e306ecf908b1ad5e/generation_config.json +[INFO|configuration_utils.py:1014] 2026-04-29 15:39:13,129 >> Generate config GenerationConfig { + "bos_token_id": 151643, + "do_sample": true, + "eos_token_id": [ + 151645, + 151643 + ], + "pad_token_id": 151643, + "temperature": 0.6, + "top_k": 20, + "top_p": 0.95 +} + +The tokenizer has new PAD/BOS/EOS tokens that differ from the model config and generation config. The model config and generation config were aligned accordingly, being updated with the tokenizer's values. Updated tokens: {'bos_token_id': None, 'pad_token_id': 151643}. +The tied weights mapping and config for this model specifies to tie model.embed_tokens.weight to lm_head.weight, but both are present in the checkpoints, so we will NOT tie them. You should update the config with `tie_word_embeddings=False` to silence this warning +ywang29-p4d-debug-1-worker-0:213684:213684 [3] NCCL INFO Comm config Blocking set to 1 +The tied weights mapping and config for this model specifies to tie model.embed_tokens.weight to lm_head.weight, but both are present in the checkpoints, so we will NOT tie them. You should update the config with `tie_word_embeddings=False` to silence this warning +[INFO|dynamic_module_utils.py:406] 2026-04-29 15:39:13,231 >> Could not locate the custom_generate/generate.py inside Qwen/Qwen3-1.7B. +[INFO|2026-04-29 15:39:13] llamafactory.model.model_utils.checkpointing:144 >> Gradient checkpointing enabled. +[INFO|2026-04-29 15:39:13] llamafactory.model.model_utils.attention:144 >> Using torch SDPA for faster training and inference. +[INFO|2026-04-29 15:39:13] llamafactory.model.adapter:144 >> Upcasting trainable params to float32. +[INFO|2026-04-29 15:39:13] llamafactory.model.adapter:144 >> Fine-tuning method: Full +[INFO|2026-04-29 15:39:13] llamafactory.model.loader:144 >> trainable params: 2,031,739,904 || all params: 2,031,739,904 || trainable%: 100.0000 +The tokenizer has new PAD/BOS/EOS tokens that differ from the model config and generation config. The model config and generation config were aligned accordingly, being updated with the tokenizer's values. Updated tokens: {'bos_token_id': None, 'pad_token_id': 151643}. +ywang29-p4d-debug-1-worker-0:213687:213687 [6] NCCL INFO Comm config Blocking set to 1 +The tokenizer has new PAD/BOS/EOS tokens that differ from the model config and generation config. The model config and generation config were aligned accordingly, being updated with the tokenizer's values. Updated tokens: {'bos_token_id': None, 'pad_token_id': 151643}. +The tokenizer has new PAD/BOS/EOS tokens that differ from the model config and generation config. The model config and generation config were aligned accordingly, being updated with the tokenizer's values. Updated tokens: {'bos_token_id': None, 'pad_token_id': 151643}. +ywang29-p4d-debug-1-worker-0:213686:213686 [5] NCCL INFO Comm config Blocking set to 1 +ywang29-p4d-debug-1-worker-0:213682:213682 [1] NCCL INFO Comm config Blocking set to 1 +[WARNING|trainer_utils.py:1234] 2026-04-29 15:39:13,812 >> The tokenizer has new PAD/BOS/EOS tokens that differ from the model config and generation config. The model config and generation config were aligned accordingly, being updated with the tokenizer's values. Updated tokens: {'bos_token_id': None, 'pad_token_id': 151643}. +ywang29-p4d-debug-1-worker-0:213688:213688 [7] NCCL INFO Comm config Blocking set to 1 +ywang29-p4d-debug-1-worker-0:213685:213685 [4] NCCL INFO Comm config Blocking set to 1 +ywang29-p4d-debug-1-worker-0:213683:213683 [2] NCCL INFO Comm config Blocking set to 1 +ywang29-p4d-debug-1-worker-0:213681:213681 [0] NCCL INFO Comm config Blocking set to 1 +ywang29-p4d-debug-1-worker-0:213681:213681 [0] NCCL INFO Initialized NET plugin Socket +ywang29-p4d-debug-1-worker-0:213683:213683 [2] NCCL INFO Initialized NET plugin Socket +ywang29-p4d-debug-1-worker-0:213684:213684 [3] NCCL INFO Initialized NET plugin Socket +ywang29-p4d-debug-1-worker-0:213685:213685 [4] NCCL INFO Initialized NET plugin Socket +ywang29-p4d-debug-1-worker-0:213688:213688 [7] NCCL INFO Initialized NET plugin Socket +ywang29-p4d-debug-1-worker-0:213687:213687 [6] NCCL INFO Initialized NET plugin Socket +ywang29-p4d-debug-1-worker-0:213686:213686 [5] NCCL INFO Initialized NET plugin Socket +ywang29-p4d-debug-1-worker-0:213682:213682 [1] NCCL INFO Initialized NET plugin Socket +ywang29-p4d-debug-1-worker-0:213683:213683 [2] NCCL INFO Assigned NET plugin Socket to comm +ywang29-p4d-debug-1-worker-0:213684:213684 [3] NCCL INFO Assigned NET plugin Socket to comm +ywang29-p4d-debug-1-worker-0:213681:213681 [0] NCCL INFO Assigned NET plugin Socket to comm +ywang29-p4d-debug-1-worker-0:213683:213683 [2] NCCL INFO Using network Socket +ywang29-p4d-debug-1-worker-0:213688:213688 [7] NCCL INFO Assigned NET plugin Socket to comm +ywang29-p4d-debug-1-worker-0:213684:213684 [3] NCCL INFO Using network Socket +ywang29-p4d-debug-1-worker-0:213681:213681 [0] NCCL INFO Using network Socket +ywang29-p4d-debug-1-worker-0:213687:213687 [6] NCCL INFO Assigned NET plugin Socket to comm +ywang29-p4d-debug-1-worker-0:213688:213688 [7] NCCL INFO Using network Socket +ywang29-p4d-debug-1-worker-0:213685:213685 [4] NCCL INFO Assigned NET plugin Socket to comm +ywang29-p4d-debug-1-worker-0:213687:213687 [6] NCCL INFO Using network Socket +ywang29-p4d-debug-1-worker-0:213685:213685 [4] NCCL INFO Using network Socket +ywang29-p4d-debug-1-worker-0:213682:213682 [1] NCCL INFO Assigned NET plugin Socket to comm +ywang29-p4d-debug-1-worker-0:213686:213686 [5] NCCL INFO Assigned NET plugin Socket to comm +ywang29-p4d-debug-1-worker-0:213682:213682 [1] NCCL INFO Using network Socket +ywang29-p4d-debug-1-worker-0:213686:213686 [5] NCCL INFO Using network Socket +ywang29-p4d-debug-1-worker-0:213684:213684 [3] NCCL INFO ncclCommSplit comm 0x5616c2ee4db0 rank 3 nranks 8 cudaDev 3 nvmlDev 3 busId 201d0 parent 0x5616b5c85c30 splitCount 1 color 1266629538 key 3- Init START +ywang29-p4d-debug-1-worker-0:213685:213685 [4] NCCL INFO ncclCommSplit comm 0x55cfdde9b730 rank 4 nranks 8 cudaDev 4 nvmlDev 4 busId 901c0 parent 0x55cfd20e4910 splitCount 1 color 1266629538 key 4- Init START +ywang29-p4d-debug-1-worker-0:213688:213688 [7] NCCL INFO ncclCommSplit comm 0x55e5e87890a0 rank 7 nranks 8 cudaDev 7 nvmlDev 7 busId a01d0 parent 0x55e5dac15dd0 splitCount 1 color 1266629538 key 7- Init START +ywang29-p4d-debug-1-worker-0:213686:213686 [5] NCCL INFO ncclCommSplit comm 0x55cae6fe89c0 rank 5 nranks 8 cudaDev 5 nvmlDev 5 busId 901d0 parent 0x55cadade89e0 splitCount 1 color 1266629538 key 5- Init START +ywang29-p4d-debug-1-worker-0:213683:213683 [2] NCCL INFO ncclCommSplit comm 0x55eca298cda0 rank 2 nranks 8 cudaDev 2 nvmlDev 2 busId 201c0 parent 0x55ec9c119990 splitCount 1 color 1266629538 key 2- Init START +ywang29-p4d-debug-1-worker-0:213687:213687 [6] NCCL INFO ncclCommSplit comm 0x55e875022b30 rank 6 nranks 8 cudaDev 6 nvmlDev 6 busId a01c0 parent 0x55e8675b14b0 splitCount 1 color 1266629538 key 6- Init START +ywang29-p4d-debug-1-worker-0:213682:213682 [1] NCCL INFO ncclCommSplit comm 0x557309603670 rank 1 nranks 8 cudaDev 1 nvmlDev 1 busId 101d0 parent 0x55730064a610 splitCount 1 color 1266629538 key 1- Init START +ywang29-p4d-debug-1-worker-0:213681:213681 [0] NCCL INFO ncclCommSplit comm 0x5635f6273520 rank 0 nranks 8 cudaDev 0 nvmlDev 0 busId 101c0 parent 0x5635e92c0280 splitCount 1 color 1266629538 key 0- Init START +ywang29-p4d-debug-1-worker-0:213688:213688 [7] NCCL INFO ncclTopoGetCpuAffinity: Affinity for GPU 7 is 24-47,72-95. (GPU affinity = 24-47,72-95 ; CPU affinity = 0-95). +ywang29-p4d-debug-1-worker-0:213688:213688 [7] NCCL INFO NVLS multicast support is not available on dev 7 (NVLS_NCHANNELS 0) +ywang29-p4d-debug-1-worker-0:213687:213687 [6] NCCL INFO ncclTopoGetCpuAffinity: Affinity for GPU 6 is 24-47,72-95. (GPU affinity = 24-47,72-95 ; CPU affinity = 0-95). +ywang29-p4d-debug-1-worker-0:213687:213687 [6] NCCL INFO NVLS multicast support is not available on dev 6 (NVLS_NCHANNELS 0) +ywang29-p4d-debug-1-worker-0:213686:213686 [5] NCCL INFO ncclTopoGetCpuAffinity: Affinity for GPU 5 is 24-47,72-95. (GPU affinity = 24-47,72-95 ; CPU affinity = 0-95). +ywang29-p4d-debug-1-worker-0:213686:213686 [5] NCCL INFO NVLS multicast support is not available on dev 5 (NVLS_NCHANNELS 0) +ywang29-p4d-debug-1-worker-0:213682:213682 [1] NCCL INFO ncclTopoGetCpuAffinity: Affinity for GPU 1 is 0-23,48-71. (GPU affinity = 0-23,48-71 ; CPU affinity = 0-95). +ywang29-p4d-debug-1-worker-0:213682:213682 [1] NCCL INFO NVLS multicast support is not available on dev 1 (NVLS_NCHANNELS 0) +ywang29-p4d-debug-1-worker-0:213684:213684 [3] NCCL INFO ncclTopoGetCpuAffinity: Affinity for GPU 3 is 0-23,48-71. (GPU affinity = 0-23,48-71 ; CPU affinity = 0-95). +ywang29-p4d-debug-1-worker-0:213684:213684 [3] NCCL INFO NVLS multicast support is not available on dev 3 (NVLS_NCHANNELS 0) +ywang29-p4d-debug-1-worker-0:213681:213681 [0] NCCL INFO ncclTopoGetCpuAffinity: Affinity for GPU 0 is 0-23,48-71. (GPU affinity = 0-23,48-71 ; CPU affinity = 0-95). +ywang29-p4d-debug-1-worker-0:213681:213681 [0] NCCL INFO NVLS multicast support is not available on dev 0 (NVLS_NCHANNELS 0) +ywang29-p4d-debug-1-worker-0:213685:213685 [4] NCCL INFO ncclTopoGetCpuAffinity: Affinity for GPU 4 is 24-47,72-95. (GPU affinity = 24-47,72-95 ; CPU affinity = 0-95). +ywang29-p4d-debug-1-worker-0:213685:213685 [4] NCCL INFO NVLS multicast support is not available on dev 4 (NVLS_NCHANNELS 0) +ywang29-p4d-debug-1-worker-0:213683:213683 [2] NCCL INFO ncclTopoGetCpuAffinity: Affinity for GPU 2 is 0-23,48-71. (GPU affinity = 0-23,48-71 ; CPU affinity = 0-95). +ywang29-p4d-debug-1-worker-0:213683:213683 [2] NCCL INFO NVLS multicast support is not available on dev 2 (NVLS_NCHANNELS 0) +ywang29-p4d-debug-1-worker-0:213683:213683 [2] NCCL INFO comm 0x55eca298cda0 rank 2 nRanks 8 nNodes 1 localRanks 8 localRank 2 MNNVL 0 +ywang29-p4d-debug-1-worker-0:213682:213682 [1] NCCL INFO comm 0x557309603670 rank 1 nRanks 8 nNodes 1 localRanks 8 localRank 1 MNNVL 0 +ywang29-p4d-debug-1-worker-0:213681:213681 [0] NCCL INFO comm 0x5635f6273520 rank 0 nRanks 8 nNodes 1 localRanks 8 localRank 0 MNNVL 0 +ywang29-p4d-debug-1-worker-0:213686:213686 [5] NCCL INFO comm 0x55cae6fe89c0 rank 5 nRanks 8 nNodes 1 localRanks 8 localRank 5 MNNVL 0 +ywang29-p4d-debug-1-worker-0:213687:213687 [6] NCCL INFO comm 0x55e875022b30 rank 6 nRanks 8 nNodes 1 localRanks 8 localRank 6 MNNVL 0 +ywang29-p4d-debug-1-worker-0:213685:213685 [4] NCCL INFO comm 0x55cfdde9b730 rank 4 nRanks 8 nNodes 1 localRanks 8 localRank 4 MNNVL 0 +ywang29-p4d-debug-1-worker-0:213688:213688 [7] NCCL INFO comm 0x55e5e87890a0 rank 7 nRanks 8 nNodes 1 localRanks 8 localRank 7 MNNVL 0 +ywang29-p4d-debug-1-worker-0:213684:213684 [3] NCCL INFO comm 0x5616c2ee4db0 rank 3 nRanks 8 nNodes 1 localRanks 8 localRank 3 MNNVL 0 +ywang29-p4d-debug-1-worker-0:213681:213681 [0] NCCL INFO Channel 00/24 : 0 1 2 3 4 5 6 7 +ywang29-p4d-debug-1-worker-0:213681:213681 [0] NCCL INFO Channel 01/24 : 0 1 2 3 4 5 6 7 +ywang29-p4d-debug-1-worker-0:213681:213681 [0] NCCL INFO Channel 02/24 : 0 1 2 3 4 5 6 7 +ywang29-p4d-debug-1-worker-0:213681:213681 [0] NCCL INFO Channel 03/24 : 0 1 2 3 4 5 6 7 +ywang29-p4d-debug-1-worker-0:213681:213681 [0] NCCL INFO Channel 04/24 : 0 1 2 3 4 5 6 7 +ywang29-p4d-debug-1-worker-0:213681:213681 [0] NCCL INFO Channel 05/24 : 0 1 2 3 4 5 6 7 +ywang29-p4d-debug-1-worker-0:213683:213683 [2] NCCL INFO Trees [0] 3/-1/-1->2->1 [1] 3/-1/-1->2->1 [2] 3/-1/-1->2->1 [3] 3/-1/-1->2->1 [4] 3/-1/-1->2->1 [5] 3/-1/-1->2->1 [6] 3/-1/-1->2->1 [7] 3/-1/-1->2->1 [8] 3/-1/-1->2->1 [9] 3/-1/-1->2->1 [10] 3/-1/-1->2->1 [11] 3/-1/-1->2->1 [12] 3/-1/-1->2->1 [13] 3/-1/-1->2->1 [14] 3/-1/-1->2->1 [15] 3/-1/-1->2->1 [16] 3/-1/-1->2->1 [17] 3/-1/-1->2->1 [18] 3/-1/-1->2->1 [19] 3/-1/-1->2->1 [20] 3/-1/-1->2->1 [21] 3/-1/-1->2->1 [22] 3/-1/-1->2->1 [23] 3/-1/-1->2->1 +ywang29-p4d-debug-1-worker-0:213681:213681 [0] NCCL INFO Channel 06/24 : 0 1 2 3 4 5 6 7 +ywang29-p4d-debug-1-worker-0:213683:213683 [2] NCCL INFO P2P Chunksize set to 524288 +ywang29-p4d-debug-1-worker-0:213681:213681 [0] NCCL INFO Channel 07/24 : 0 1 2 3 4 5 6 7 +ywang29-p4d-debug-1-worker-0:213682:213682 [1] NCCL INFO Trees [0] 2/-1/-1->1->0 [1] 2/-1/-1->1->0 [2] 2/-1/-1->1->0 [3] 2/-1/-1->1->0 [4] 2/-1/-1->1->0 [5] 2/-1/-1->1->0 [6] 2/-1/-1->1->0 [7] 2/-1/-1->1->0 [8] 2/-1/-1->1->0 [9] 2/-1/-1->1->0 [10] 2/-1/-1->1->0 [11] 2/-1/-1->1->0 [12] 2/-1/-1->1->0 [13] 2/-1/-1->1->0 [14] 2/-1/-1->1->0 [15] 2/-1/-1->1->0 [16] 2/-1/-1->1->0 [17] 2/-1/-1->1->0 [18] 2/-1/-1->1->0 [19] 2/-1/-1->1->0 [20] 2/-1/-1->1->0 [21] 2/-1/-1->1->0 [22] 2/-1/-1->1->0 [23] 2/-1/-1->1->0 +ywang29-p4d-debug-1-worker-0:213681:213681 [0] NCCL INFO Channel 08/24 : 0 1 2 3 4 5 6 7 +ywang29-p4d-debug-1-worker-0:213682:213682 [1] NCCL INFO P2P Chunksize set to 524288 +ywang29-p4d-debug-1-worker-0:213686:213686 [5] NCCL INFO Trees [0] 6/-1/-1->5->4 [1] 6/-1/-1->5->4 [2] 6/-1/-1->5->4 [3] 6/-1/-1->5->4 [4] 6/-1/-1->5->4 [5] 6/-1/-1->5->4 [6] 6/-1/-1->5->4 [7] 6/-1/-1->5->4 [8] 6/-1/-1->5->4 [9] 6/-1/-1->5->4 [10] 6/-1/-1->5->4 [11] 6/-1/-1->5->4 [12] 6/-1/-1->5->4 [13] 6/-1/-1->5->4 [14] 6/-1/-1->5->4 [15] 6/-1/-1->5->4 [16] 6/-1/-1->5->4 [17] 6/-1/-1->5->4 [18] 6/-1/-1->5->4 [19] 6/-1/-1->5->4 [20] 6/-1/-1->5->4 [21] 6/-1/-1->5->4 [22] 6/-1/-1->5->4 [23] 6/-1/-1->5->4 +ywang29-p4d-debug-1-worker-0:213681:213681 [0] NCCL INFO Channel 09/24 : 0 1 2 3 4 5 6 7 +ywang29-p4d-debug-1-worker-0:213687:213687 [6] NCCL INFO Trees [0] 7/-1/-1->6->5 [1] 7/-1/-1->6->5 [2] 7/-1/-1->6->5 [3] 7/-1/-1->6->5 [4] 7/-1/-1->6->5 [5] 7/-1/-1->6->5 [6] 7/-1/-1->6->5 [7] 7/-1/-1->6->5 [8] 7/-1/-1->6->5 [9] 7/-1/-1->6->5 [10] 7/-1/-1->6->5 [11] 7/-1/-1->6->5 [12] 7/-1/-1->6->5 [13] 7/-1/-1->6->5 [14] 7/-1/-1->6->5 [15] 7/-1/-1->6->5 [16] 7/-1/-1->6->5 [17] 7/-1/-1->6->5 [18] 7/-1/-1->6->5 [19] 7/-1/-1->6->5 [20] 7/-1/-1->6->5 [21] 7/-1/-1->6->5 [22] 7/-1/-1->6->5 [23] 7/-1/-1->6->5 +ywang29-p4d-debug-1-worker-0:213686:213686 [5] NCCL INFO P2P Chunksize set to 524288 +ywang29-p4d-debug-1-worker-0:213685:213685 [4] NCCL INFO Trees [0] 5/-1/-1->4->3 [1] 5/-1/-1->4->3 [2] 5/-1/-1->4->3 [3] 5/-1/-1->4->3 [4] 5/-1/-1->4->3 [5] 5/-1/-1->4->3 [6] 5/-1/-1->4->3 [7] 5/-1/-1->4->3 [8] 5/-1/-1->4->3 [9] 5/-1/-1->4->3 [10] 5/-1/-1->4->3 [11] 5/-1/-1->4->3 [12] 5/-1/-1->4->3 [13] 5/-1/-1->4->3 [14] 5/-1/-1->4->3 [15] 5/-1/-1->4->3 [16] 5/-1/-1->4->3 [17] 5/-1/-1->4->3 [18] 5/-1/-1->4->3 [19] 5/-1/-1->4->3 [20] 5/-1/-1->4->3 [21] 5/-1/-1->4->3 [22] 5/-1/-1->4->3 [23] 5/-1/-1->4->3 +ywang29-p4d-debug-1-worker-0:213687:213687 [6] NCCL INFO P2P Chunksize set to 524288 +ywang29-p4d-debug-1-worker-0:213681:213681 [0] NCCL INFO Channel 10/24 : 0 1 2 3 4 5 6 7 +ywang29-p4d-debug-1-worker-0:213684:213684 [3] NCCL INFO Trees [0] 4/-1/-1->3->2 [1] 4/-1/-1->3->2 [2] 4/-1/-1->3->2 [3] 4/-1/-1->3->2 [4] 4/-1/-1->3->2 [5] 4/-1/-1->3->2 [6] 4/-1/-1->3->2 [7] 4/-1/-1->3->2 [8] 4/-1/-1->3->2 [9] 4/-1/-1->3->2 [10] 4/-1/-1->3->2 [11] 4/-1/-1->3->2 [12] 4/-1/-1->3->2 [13] 4/-1/-1->3->2 [14] 4/-1/-1->3->2 [15] 4/-1/-1->3->2 [16] 4/-1/-1->3->2 [17] 4/-1/-1->3->2 [18] 4/-1/-1->3->2 [19] 4/-1/-1->3->2 [20] 4/-1/-1->3->2 [21] 4/-1/-1->3->2 [22] 4/-1/-1->3->2 [23] 4/-1/-1->3->2 +ywang29-p4d-debug-1-worker-0:213685:213685 [4] NCCL INFO P2P Chunksize set to 524288 +ywang29-p4d-debug-1-worker-0:213681:213681 [0] NCCL INFO Channel 11/24 : 0 1 2 3 4 5 6 7 +ywang29-p4d-debug-1-worker-0:213688:213688 [7] NCCL INFO Trees [0] -1/-1/-1->7->6 [1] -1/-1/-1->7->6 [2] -1/-1/-1->7->6 [3] -1/-1/-1->7->6 [4] -1/-1/-1->7->6 [5] -1/-1/-1->7->6 [6] -1/-1/-1->7->6 [7] -1/-1/-1->7->6 [8] -1/-1/-1->7->6 [9] -1/-1/-1->7->6 [10] -1/-1/-1->7->6 [11] -1/-1/-1->7->6 [12] -1/-1/-1->7->6 [13] -1/-1/-1->7->6 [14] -1/-1/-1->7->6 [15] -1/-1/-1->7->6 [16] -1/-1/-1->7->6 [17] -1/-1/-1->7->6 [18] -1/-1/-1->7->6 [19] -1/-1/-1->7->6 [20] -1/-1/-1->7->6 [21] -1/-1/-1->7->6 [22] -1/-1/-1->7->6 [23] -1/-1/-1->7->6 +ywang29-p4d-debug-1-worker-0:213684:213684 [3] NCCL INFO P2P Chunksize set to 524288 +ywang29-p4d-debug-1-worker-0:213681:213681 [0] NCCL INFO Channel 12/24 : 0 1 2 3 4 5 6 7 +ywang29-p4d-debug-1-worker-0:213688:213688 [7] NCCL INFO P2P Chunksize set to 524288 +ywang29-p4d-debug-1-worker-0:213681:213681 [0] NCCL INFO Channel 13/24 : 0 1 2 3 4 5 6 7 +ywang29-p4d-debug-1-worker-0:213681:213681 [0] NCCL INFO Channel 14/24 : 0 1 2 3 4 5 6 7 +ywang29-p4d-debug-1-worker-0:213681:213681 [0] NCCL INFO Channel 15/24 : 0 1 2 3 4 5 6 7 +ywang29-p4d-debug-1-worker-0:213681:213681 [0] NCCL INFO Channel 16/24 : 0 1 2 3 4 5 6 7 +ywang29-p4d-debug-1-worker-0:213681:213681 [0] NCCL INFO Channel 17/24 : 0 1 2 3 4 5 6 7 +ywang29-p4d-debug-1-worker-0:213681:213681 [0] NCCL INFO Channel 18/24 : 0 1 2 3 4 5 6 7 +ywang29-p4d-debug-1-worker-0:213681:213681 [0] NCCL INFO Channel 19/24 : 0 1 2 3 4 5 6 7 +ywang29-p4d-debug-1-worker-0:213681:213681 [0] NCCL INFO Channel 20/24 : 0 1 2 3 4 5 6 7 +ywang29-p4d-debug-1-worker-0:213681:213681 [0] NCCL INFO Channel 21/24 : 0 1 2 3 4 5 6 7 +ywang29-p4d-debug-1-worker-0:213681:213681 [0] NCCL INFO Channel 22/24 : 0 1 2 3 4 5 6 7 +ywang29-p4d-debug-1-worker-0:213681:213681 [0] NCCL INFO Channel 23/24 : 0 1 2 3 4 5 6 7 +ywang29-p4d-debug-1-worker-0:213681:213681 [0] NCCL INFO Trees [0] 1/-1/-1->0->-1 [1] 1/-1/-1->0->-1 [2] 1/-1/-1->0->-1 [3] 1/-1/-1->0->-1 [4] 1/-1/-1->0->-1 [5] 1/-1/-1->0->-1 [6] 1/-1/-1->0->-1 [7] 1/-1/-1->0->-1 [8] 1/-1/-1->0->-1 [9] 1/-1/-1->0->-1 [10] 1/-1/-1->0->-1 [11] 1/-1/-1->0->-1 [12] 1/-1/-1->0->-1 [13] 1/-1/-1->0->-1 [14] 1/-1/-1->0->-1 [15] 1/-1/-1->0->-1 [16] 1/-1/-1->0->-1 [17] 1/-1/-1->0->-1 [18] 1/-1/-1->0->-1 [19] 1/-1/-1->0->-1 [20] 1/-1/-1->0->-1 [21] 1/-1/-1->0->-1 [22] 1/-1/-1->0->-1 [23] 1/-1/-1->0->-1 +ywang29-p4d-debug-1-worker-0:213681:213681 [0] NCCL INFO P2P Chunksize set to 524288 +ywang29-p4d-debug-1-worker-0:213685:213685 [4] NCCL INFO Check P2P Type isAllDirectP2p 1 directMode 0 isAllCudaP2p 1 +ywang29-p4d-debug-1-worker-0:213685:215779 [0] NCCL INFO [Proxy Service] Device 4 CPU core 75 +ywang29-p4d-debug-1-worker-0:213685:215780 [0] NCCL INFO [Proxy Service UDS] Device 4 CPU core 86 +ywang29-p4d-debug-1-worker-0:213682:213682 [1] NCCL INFO Check P2P Type isAllDirectP2p 1 directMode 0 isAllCudaP2p 1 +ywang29-p4d-debug-1-worker-0:213682:215781 [0] NCCL INFO [Proxy Service] Device 1 CPU core 3 +ywang29-p4d-debug-1-worker-0:213682:215782 [0] NCCL INFO [Proxy Service UDS] Device 1 CPU core 6 +ywang29-p4d-debug-1-worker-0:213687:213687 [6] NCCL INFO Check P2P Type isAllDirectP2p 1 directMode 0 isAllCudaP2p 1 +ywang29-p4d-debug-1-worker-0:213686:213686 [5] NCCL INFO Check P2P Type isAllDirectP2p 1 directMode 0 isAllCudaP2p 1 +ywang29-p4d-debug-1-worker-0:213687:215783 [0] NCCL INFO [Proxy Service] Device 6 CPU core 40 +ywang29-p4d-debug-1-worker-0:213687:215785 [0] NCCL INFO [Proxy Service UDS] Device 6 CPU core 44 +ywang29-p4d-debug-1-worker-0:213686:215784 [0] NCCL INFO [Proxy Service] Device 5 CPU core 43 +ywang29-p4d-debug-1-worker-0:213686:215786 [0] NCCL INFO [Proxy Service UDS] Device 5 CPU core 85 +ywang29-p4d-debug-1-worker-0:213688:213688 [7] NCCL INFO Check P2P Type isAllDirectP2p 1 directMode 0 isAllCudaP2p 1 +ywang29-p4d-debug-1-worker-0:213688:215787 [0] NCCL INFO [Proxy Service] Device 7 CPU core 81 +ywang29-p4d-debug-1-worker-0:213688:215788 [0] NCCL INFO [Proxy Service UDS] Device 7 CPU core 77 +ywang29-p4d-debug-1-worker-0:213684:213684 [3] NCCL INFO Check P2P Type isAllDirectP2p 1 directMode 0 isAllCudaP2p 1 +ywang29-p4d-debug-1-worker-0:213683:213683 [2] NCCL INFO Check P2P Type isAllDirectP2p 1 directMode 0 isAllCudaP2p 1 +ywang29-p4d-debug-1-worker-0:213684:215789 [0] NCCL INFO [Proxy Service] Device 3 CPU core 4 +ywang29-p4d-debug-1-worker-0:213683:215790 [0] NCCL INFO [Proxy Service] Device 2 CPU core 9 +ywang29-p4d-debug-1-worker-0:213683:215792 [0] NCCL INFO [Proxy Service UDS] Device 2 CPU core 59 +ywang29-p4d-debug-1-worker-0:213684:215791 [0] NCCL INFO [Proxy Service UDS] Device 3 CPU core 58 +ywang29-p4d-debug-1-worker-0:213681:213681 [0] NCCL INFO Check P2P Type isAllDirectP2p 1 directMode 0 isAllCudaP2p 1 +ywang29-p4d-debug-1-worker-0:213681:215794 [0] NCCL INFO [Proxy Service UDS] Device 0 CPU core 16 +ywang29-p4d-debug-1-worker-0:213681:215793 [0] NCCL INFO [Proxy Service] Device 0 CPU core 60 +ywang29-p4d-debug-1-worker-0:213686:213686 [5] NCCL INFO NET/OFI NCCL_OFI_TUNER is not available for platform : p4de.24xlarge, Fall back to NCCL's tuner +ywang29-p4d-debug-1-worker-0:213686:213686 [5] NCCL INFO threadThresholds 8/8/64 | 64/8/64 | 512 | 512 +ywang29-p4d-debug-1-worker-0:213686:213686 [5] NCCL INFO 24 coll channels, 24 collnet channels, 0 nvls channels, 32 p2p channels, 32 p2p channels per peer +ywang29-p4d-debug-1-worker-0:213687:213687 [6] NCCL INFO NET/OFI NCCL_OFI_TUNER is not available for platform : p4de.24xlarge, Fall back to NCCL's tuner +ywang29-p4d-debug-1-worker-0:213687:213687 [6] NCCL INFO threadThresholds 8/8/64 | 64/8/64 | 512 | 512 +ywang29-p4d-debug-1-worker-0:213687:213687 [6] NCCL INFO 24 coll channels, 24 collnet channels, 0 nvls channels, 32 p2p channels, 32 p2p channels per peer +ywang29-p4d-debug-1-worker-0:213682:213682 [1] NCCL INFO NET/OFI NCCL_OFI_TUNER is not available for platform : p4de.24xlarge, Fall back to NCCL's tuner +ywang29-p4d-debug-1-worker-0:213682:213682 [1] NCCL INFO threadThresholds 8/8/64 | 64/8/64 | 512 | 512 +ywang29-p4d-debug-1-worker-0:213682:213682 [1] NCCL INFO 24 coll channels, 24 collnet channels, 0 nvls channels, 32 p2p channels, 32 p2p channels per peer +ywang29-p4d-debug-1-worker-0:213685:213685 [4] NCCL INFO NET/OFI NCCL_OFI_TUNER is not available for platform : p4de.24xlarge, Fall back to NCCL's tuner +ywang29-p4d-debug-1-worker-0:213685:213685 [4] NCCL INFO threadThresholds 8/8/64 | 64/8/64 | 512 | 512 +ywang29-p4d-debug-1-worker-0:213685:213685 [4] NCCL INFO 24 coll channels, 24 collnet channels, 0 nvls channels, 32 p2p channels, 32 p2p channels per peer +ywang29-p4d-debug-1-worker-0:213688:213688 [7] NCCL INFO NET/OFI NCCL_OFI_TUNER is not available for platform : p4de.24xlarge, Fall back to NCCL's tuner +ywang29-p4d-debug-1-worker-0:213688:213688 [7] NCCL INFO threadThresholds 8/8/64 | 64/8/64 | 512 | 512 +ywang29-p4d-debug-1-worker-0:213688:213688 [7] NCCL INFO 24 coll channels, 24 collnet channels, 0 nvls channels, 32 p2p channels, 32 p2p channels per peer +ywang29-p4d-debug-1-worker-0:213683:213683 [2] NCCL INFO NET/OFI NCCL_OFI_TUNER is not available for platform : p4de.24xlarge, Fall back to NCCL's tuner +ywang29-p4d-debug-1-worker-0:213683:213683 [2] NCCL INFO threadThresholds 8/8/64 | 64/8/64 | 512 | 512 +ywang29-p4d-debug-1-worker-0:213683:213683 [2] NCCL INFO 24 coll channels, 24 collnet channels, 0 nvls channels, 32 p2p channels, 32 p2p channels per peer +ywang29-p4d-debug-1-worker-0:213684:213684 [3] NCCL INFO NET/OFI NCCL_OFI_TUNER is not available for platform : p4de.24xlarge, Fall back to NCCL's tuner +ywang29-p4d-debug-1-worker-0:213684:213684 [3] NCCL INFO threadThresholds 8/8/64 | 64/8/64 | 512 | 512 +ywang29-p4d-debug-1-worker-0:213684:213684 [3] NCCL INFO 24 coll channels, 24 collnet channels, 0 nvls channels, 32 p2p channels, 32 p2p channels per peer +ywang29-p4d-debug-1-worker-0:213681:213681 [0] NCCL INFO NET/OFI NCCL_OFI_TUNER is not available for platform : p4de.24xlarge, Fall back to NCCL's tuner +ywang29-p4d-debug-1-worker-0:213681:213681 [0] NCCL INFO threadThresholds 8/8/64 | 64/8/64 | 512 | 512 +ywang29-p4d-debug-1-worker-0:213681:213681 [0] NCCL INFO 24 coll channels, 24 collnet channels, 0 nvls channels, 32 p2p channels, 32 p2p channels per peer +ywang29-p4d-debug-1-worker-0:213681:213681 [0] NCCL INFO CC Off, workFifoBytes 1048576 +ywang29-p4d-debug-1-worker-0:213684:213684 [3] NCCL INFO ncclCommSplit comm 0x5616c2ee4db0 rank 3 nranks 8 cudaDev 3 nvmlDev 3 busId 201d0 parent 0x5616b5c85c30 splitCount 1 color 1266629538 key 3 - Init COMPLETE +ywang29-p4d-debug-1-worker-0:213688:213688 [7] NCCL INFO ncclCommSplit comm 0x55e5e87890a0 rank 7 nranks 8 cudaDev 7 nvmlDev 7 busId a01d0 parent 0x55e5dac15dd0 splitCount 1 color 1266629538 key 7 - Init COMPLETE +ywang29-p4d-debug-1-worker-0:213684:213684 [3] NCCL INFO Init timings - ncclCommSplit: rank 3 nranks 8 total 1.17 (kernels 0.00, alloc 0.00, bootstrap 0.00, allgathers 0.01, topo 0.10, graphs 0.00, connections 0.06, rest 1.00) +ywang29-p4d-debug-1-worker-0:213686:213686 [5] NCCL INFO ncclCommSplit comm 0x55cae6fe89c0 rank 5 nranks 8 cudaDev 5 nvmlDev 5 busId 901d0 parent 0x55cadade89e0 splitCount 1 color 1266629538 key 5 - Init COMPLETE +ywang29-p4d-debug-1-worker-0:213682:213682 [1] NCCL INFO ncclCommSplit comm 0x557309603670 rank 1 nranks 8 cudaDev 1 nvmlDev 1 busId 101d0 parent 0x55730064a610 splitCount 1 color 1266629538 key 1 - Init COMPLETE +ywang29-p4d-debug-1-worker-0:213688:213688 [7] NCCL INFO Init timings - ncclCommSplit: rank 7 nranks 8 total 0.55 (kernels 0.00, alloc 0.00, bootstrap 0.00, allgathers 0.01, topo 0.09, graphs 0.00, connections 0.06, rest 0.39) +ywang29-p4d-debug-1-worker-0:213686:213686 [5] NCCL INFO Init timings - ncclCommSplit: rank 5 nranks 8 total 0.77 (kernels 0.00, alloc 0.00, bootstrap 0.00, allgathers 0.01, topo 0.10, graphs 0.00, connections 0.05, rest 0.62) +ywang29-p4d-debug-1-worker-0:213682:213682 [1] NCCL INFO Init timings - ncclCommSplit: rank 1 nranks 8 total 0.76 (kernels 0.00, alloc 0.00, bootstrap 0.00, allgathers 0.01, topo 0.10, graphs 0.00, connections 0.06, rest 0.60) +ywang29-p4d-debug-1-worker-0:213687:213687 [6] NCCL INFO ncclCommSplit comm 0x55e875022b30 rank 6 nranks 8 cudaDev 6 nvmlDev 6 busId a01c0 parent 0x55e8675b14b0 splitCount 1 color 1266629538 key 6 - Init COMPLETE +ywang29-p4d-debug-1-worker-0:213683:213683 [2] NCCL INFO ncclCommSplit comm 0x55eca298cda0 rank 2 nranks 8 cudaDev 2 nvmlDev 2 busId 201c0 parent 0x55ec9c119990 splitCount 1 color 1266629538 key 2 - Init COMPLETE +ywang29-p4d-debug-1-worker-0:213685:213685 [4] NCCL INFO ncclCommSplit comm 0x55cfdde9b730 rank 4 nranks 8 cudaDev 4 nvmlDev 4 busId 901c0 parent 0x55cfd20e4910 splitCount 1 color 1266629538 key 4 - Init COMPLETE +ywang29-p4d-debug-1-worker-0:213687:213687 [6] NCCL INFO Init timings - ncclCommSplit: rank 6 nranks 8 total 0.84 (kernels 0.00, alloc 0.00, bootstrap 0.00, allgathers 0.01, topo 0.10, graphs 0.00, connections 0.05, rest 0.68) +ywang29-p4d-debug-1-worker-0:213681:213681 [0] NCCL INFO ncclCommSplit comm 0x5635f6273520 rank 0 nranks 8 cudaDev 0 nvmlDev 0 busId 101c0 parent 0x5635e92c0280 splitCount 1 color 1266629538 key 0 - Init COMPLETE +ywang29-p4d-debug-1-worker-0:213683:213683 [2] NCCL INFO Init timings - ncclCommSplit: rank 2 nranks 8 total 0.33 (kernels 0.00, alloc 0.00, bootstrap 0.00, allgathers 0.00, topo 0.10, graphs 0.00, connections 0.06, rest 0.17) +ywang29-p4d-debug-1-worker-0:213685:213685 [4] NCCL INFO Init timings - ncclCommSplit: rank 4 nranks 8 total 0.36 (kernels 0.00, alloc 0.00, bootstrap 0.00, allgathers 0.00, topo 0.10, graphs 0.00, connections 0.06, rest 0.19) +ywang29-p4d-debug-1-worker-0:213681:213681 [0] NCCL INFO Init timings - ncclCommSplit: rank 0 nranks 8 total 0.19 (kernels 0.00, alloc 0.00, bootstrap 0.00, allgathers 0.00, topo 0.10, graphs 0.00, connections 0.06, rest 0.02) +ywang29-p4d-debug-1-worker-0:213682:213682 [1] NCCL INFO Channel 00/0 : 1[1] -> 2[2] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213683:213683 [2] NCCL INFO Channel 00/0 : 2[2] -> 3[3] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213684:213684 [3] NCCL INFO Channel 00/0 : 3[3] -> 4[4] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213683:213683 [2] NCCL INFO Channel 01/0 : 2[2] -> 3[3] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213688:213688 [7] NCCL INFO Channel 00/0 : 7[7] -> 0[0] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213682:213682 [1] NCCL INFO Channel 01/0 : 1[1] -> 2[2] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213682:213682 [1] NCCL INFO Channel 02/0 : 1[1] -> 2[2] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213688:213688 [7] NCCL INFO Channel 01/0 : 7[7] -> 0[0] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213684:213684 [3] NCCL INFO Channel 01/0 : 3[3] -> 4[4] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213682:213682 [1] NCCL INFO Channel 03/0 : 1[1] -> 2[2] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213683:213683 [2] NCCL INFO Channel 02/0 : 2[2] -> 3[3] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213686:213686 [5] NCCL INFO Channel 00/0 : 5[5] -> 6[6] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213685:213685 [4] NCCL INFO Channel 00/0 : 4[4] -> 5[5] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213688:213688 [7] NCCL INFO Channel 02/0 : 7[7] -> 0[0] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213688:213688 [7] NCCL INFO Channel 03/0 : 7[7] -> 0[0] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213682:213682 [1] NCCL INFO Channel 04/0 : 1[1] -> 2[2] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213684:213684 [3] NCCL INFO Channel 02/0 : 3[3] -> 4[4] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213687:213687 [6] NCCL INFO Channel 00/0 : 6[6] -> 7[7] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213685:213685 [4] NCCL INFO Channel 01/0 : 4[4] -> 5[5] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213686:213686 [5] NCCL INFO Channel 01/0 : 5[5] -> 6[6] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213684:213684 [3] NCCL INFO Channel 03/0 : 3[3] -> 4[4] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213685:213685 [4] NCCL INFO Channel 02/0 : 4[4] -> 5[5] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213683:213683 [2] NCCL INFO Channel 03/0 : 2[2] -> 3[3] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213688:213688 [7] NCCL INFO Channel 04/0 : 7[7] -> 0[0] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213687:213687 [6] NCCL INFO Channel 01/0 : 6[6] -> 7[7] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213686:213686 [5] NCCL INFO Channel 02/0 : 5[5] -> 6[6] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213686:213686 [5] NCCL INFO Channel 03/0 : 5[5] -> 6[6] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213688:213688 [7] NCCL INFO Channel 05/0 : 7[7] -> 0[0] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213682:213682 [1] NCCL INFO Channel 05/0 : 1[1] -> 2[2] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213687:213687 [6] NCCL INFO Channel 02/0 : 6[6] -> 7[7] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213683:213683 [2] NCCL INFO Channel 04/0 : 2[2] -> 3[3] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213684:213684 [3] NCCL INFO Channel 04/0 : 3[3] -> 4[4] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213683:213683 [2] NCCL INFO Channel 05/0 : 2[2] -> 3[3] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213685:213685 [4] NCCL INFO Channel 03/0 : 4[4] -> 5[5] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213686:213686 [5] NCCL INFO Channel 04/0 : 5[5] -> 6[6] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213684:213684 [3] NCCL INFO Channel 05/0 : 3[3] -> 4[4] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213687:213687 [6] NCCL INFO Channel 03/0 : 6[6] -> 7[7] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213681:213681 [0] NCCL INFO Channel 00/0 : 0[0] -> 1[1] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213688:213688 [7] NCCL INFO Channel 06/0 : 7[7] -> 0[0] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213688:213688 [7] NCCL INFO Channel 07/0 : 7[7] -> 0[0] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213683:213683 [2] NCCL INFO Channel 06/0 : 2[2] -> 3[3] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213687:213687 [6] NCCL INFO Channel 04/0 : 6[6] -> 7[7] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213681:213681 [0] NCCL INFO Channel 01/0 : 0[0] -> 1[1] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213682:213682 [1] NCCL INFO Channel 06/0 : 1[1] -> 2[2] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213684:213684 [3] NCCL INFO Channel 06/0 : 3[3] -> 4[4] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213682:213682 [1] NCCL INFO Channel 07/0 : 1[1] -> 2[2] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213685:213685 [4] NCCL INFO Channel 04/0 : 4[4] -> 5[5] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213681:213681 [0] NCCL INFO Channel 02/0 : 0[0] -> 1[1] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213687:213687 [6] NCCL INFO Channel 05/0 : 6[6] -> 7[7] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213686:213686 [5] NCCL INFO Channel 05/0 : 5[5] -> 6[6] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213688:213688 [7] NCCL INFO Channel 08/0 : 7[7] -> 0[0] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213681:213681 [0] NCCL INFO Channel 03/0 : 0[0] -> 1[1] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213685:213685 [4] NCCL INFO Channel 05/0 : 4[4] -> 5[5] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213684:213684 [3] NCCL INFO Channel 07/0 : 3[3] -> 4[4] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213686:213686 [5] NCCL INFO Channel 06/0 : 5[5] -> 6[6] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213687:213687 [6] NCCL INFO Channel 06/0 : 6[6] -> 7[7] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213682:213682 [1] NCCL INFO Channel 08/0 : 1[1] -> 2[2] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213686:213686 [5] NCCL INFO Channel 07/0 : 5[5] -> 6[6] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213683:213683 [2] NCCL INFO Channel 07/0 : 2[2] -> 3[3] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213687:213687 [6] NCCL INFO Channel 07/0 : 6[6] -> 7[7] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213682:213682 [1] NCCL INFO Channel 09/0 : 1[1] -> 2[2] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213684:213684 [3] NCCL INFO Channel 08/0 : 3[3] -> 4[4] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213688:213688 [7] NCCL INFO Channel 09/0 : 7[7] -> 0[0] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213687:213687 [6] NCCL INFO Channel 08/0 : 6[6] -> 7[7] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213681:213681 [0] NCCL INFO Channel 04/0 : 0[0] -> 1[1] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213685:213685 [4] NCCL INFO Channel 06/0 : 4[4] -> 5[5] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213682:213682 [1] NCCL INFO Channel 10/0 : 1[1] -> 2[2] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213683:213683 [2] NCCL INFO Channel 08/0 : 2[2] -> 3[3] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213684:213684 [3] NCCL INFO Channel 09/0 : 3[3] -> 4[4] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213682:213682 [1] NCCL INFO Channel 11/0 : 1[1] -> 2[2] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213686:213686 [5] NCCL INFO Channel 08/0 : 5[5] -> 6[6] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213683:213683 [2] NCCL INFO Channel 09/0 : 2[2] -> 3[3] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213688:213688 [7] NCCL INFO Channel 10/0 : 7[7] -> 0[0] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213685:213685 [4] NCCL INFO Channel 07/0 : 4[4] -> 5[5] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213683:213683 [2] NCCL INFO Channel 10/0 : 2[2] -> 3[3] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213681:213681 [0] NCCL INFO Channel 05/0 : 0[0] -> 1[1] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213688:213688 [7] NCCL INFO Channel 11/0 : 7[7] -> 0[0] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213684:213684 [3] NCCL INFO Channel 10/0 : 3[3] -> 4[4] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213685:213685 [4] NCCL INFO Channel 08/0 : 4[4] -> 5[5] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213686:213686 [5] NCCL INFO Channel 09/0 : 5[5] -> 6[6] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213687:213687 [6] NCCL INFO Channel 09/0 : 6[6] -> 7[7] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213687:213687 [6] NCCL INFO Channel 10/0 : 6[6] -> 7[7] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213682:213682 [1] NCCL INFO Channel 12/0 : 1[1] -> 2[2] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213685:213685 [4] NCCL INFO Channel 09/0 : 4[4] -> 5[5] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213683:213683 [2] NCCL INFO Channel 11/0 : 2[2] -> 3[3] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213681:213681 [0] NCCL INFO Channel 06/0 : 0[0] -> 1[1] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213686:213686 [5] NCCL INFO Channel 10/0 : 5[5] -> 6[6] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213684:213684 [3] NCCL INFO Channel 11/0 : 3[3] -> 4[4] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213681:213681 [0] NCCL INFO Channel 07/0 : 0[0] -> 1[1] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213686:213686 [5] NCCL INFO Channel 11/0 : 5[5] -> 6[6] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213684:213684 [3] NCCL INFO Channel 12/0 : 3[3] -> 4[4] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213688:213688 [7] NCCL INFO Channel 12/0 : 7[7] -> 0[0] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213687:213687 [6] NCCL INFO Channel 11/0 : 6[6] -> 7[7] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213687:213687 [6] NCCL INFO Channel 12/0 : 6[6] -> 7[7] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213682:213682 [1] NCCL INFO Channel 13/0 : 1[1] -> 2[2] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213688:213688 [7] NCCL INFO Channel 13/0 : 7[7] -> 0[0] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213683:213683 [2] NCCL INFO Channel 12/0 : 2[2] -> 3[3] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213685:213685 [4] NCCL INFO Channel 10/0 : 4[4] -> 5[5] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213684:213684 [3] NCCL INFO Channel 13/0 : 3[3] -> 4[4] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213681:213681 [0] NCCL INFO Channel 08/0 : 0[0] -> 1[1] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213687:213687 [6] NCCL INFO Channel 13/0 : 6[6] -> 7[7] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213683:213683 [2] NCCL INFO Channel 13/0 : 2[2] -> 3[3] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213685:213685 [4] NCCL INFO Channel 11/0 : 4[4] -> 5[5] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213688:213688 [7] NCCL INFO Channel 14/0 : 7[7] -> 0[0] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213684:213684 [3] NCCL INFO Channel 14/0 : 3[3] -> 4[4] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213686:213686 [5] NCCL INFO Channel 12/0 : 5[5] -> 6[6] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213681:213681 [0] NCCL INFO Channel 09/0 : 0[0] -> 1[1] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213684:213684 [3] NCCL INFO Channel 15/0 : 3[3] -> 4[4] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213685:213685 [4] NCCL INFO Channel 12/0 : 4[4] -> 5[5] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213682:213682 [1] NCCL INFO Channel 14/0 : 1[1] -> 2[2] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213688:213688 [7] NCCL INFO Channel 15/0 : 7[7] -> 0[0] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213683:213683 [2] NCCL INFO Channel 14/0 : 2[2] -> 3[3] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213683:213683 [2] NCCL INFO Channel 15/0 : 2[2] -> 3[3] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213687:213687 [6] NCCL INFO Channel 14/0 : 6[6] -> 7[7] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213682:213682 [1] NCCL INFO Channel 15/0 : 1[1] -> 2[2] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213688:213688 [7] NCCL INFO Channel 16/0 : 7[7] -> 0[0] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213684:213684 [3] NCCL INFO Channel 16/0 : 3[3] -> 4[4] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213685:213685 [4] NCCL INFO Channel 13/0 : 4[4] -> 5[5] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213682:213682 [1] NCCL INFO Channel 16/0 : 1[1] -> 2[2] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213688:213688 [7] NCCL INFO Channel 17/0 : 7[7] -> 0[0] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213681:213681 [0] NCCL INFO Channel 10/0 : 0[0] -> 1[1] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213685:213685 [4] NCCL INFO Channel 14/0 : 4[4] -> 5[5] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213684:213684 [3] NCCL INFO Channel 17/0 : 3[3] -> 4[4] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213686:213686 [5] NCCL INFO Channel 13/0 : 5[5] -> 6[6] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213683:213683 [2] NCCL INFO Channel 16/0 : 2[2] -> 3[3] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213687:213687 [6] NCCL INFO Channel 15/0 : 6[6] -> 7[7] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213688:213688 [7] NCCL INFO Channel 18/0 : 7[7] -> 0[0] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213682:213682 [1] NCCL INFO Channel 17/0 : 1[1] -> 2[2] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213686:213686 [5] NCCL INFO Channel 14/0 : 5[5] -> 6[6] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213684:213684 [3] NCCL INFO Channel 18/0 : 3[3] -> 4[4] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213683:213683 [2] NCCL INFO Channel 17/0 : 2[2] -> 3[3] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213685:213685 [4] NCCL INFO Channel 15/0 : 4[4] -> 5[5] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213682:213682 [1] NCCL INFO Channel 18/0 : 1[1] -> 2[2] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213686:213686 [5] NCCL INFO Channel 15/0 : 5[5] -> 6[6] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213683:213683 [2] NCCL INFO Channel 18/0 : 2[2] -> 3[3] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213684:213684 [3] NCCL INFO Channel 19/0 : 3[3] -> 4[4] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213681:213681 [0] NCCL INFO Channel 11/0 : 0[0] -> 1[1] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213681:213681 [0] NCCL INFO Channel 12/0 : 0[0] -> 1[1] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213686:213686 [5] NCCL INFO Channel 16/0 : 5[5] -> 6[6] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213688:213688 [7] NCCL INFO Channel 19/0 : 7[7] -> 0[0] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213682:213682 [1] NCCL INFO Channel 19/0 : 1[1] -> 2[2] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213687:213687 [6] NCCL INFO Channel 16/0 : 6[6] -> 7[7] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213683:213683 [2] NCCL INFO Channel 19/0 : 2[2] -> 3[3] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213685:213685 [4] NCCL INFO Channel 16/0 : 4[4] -> 5[5] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213688:213688 [7] NCCL INFO Channel 20/0 : 7[7] -> 0[0] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213684:213684 [3] NCCL INFO Channel 20/0 : 3[3] -> 4[4] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213687:213687 [6] NCCL INFO Channel 17/0 : 6[6] -> 7[7] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213682:213682 [1] NCCL INFO Channel 20/0 : 1[1] -> 2[2] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213681:213681 [0] NCCL INFO Channel 13/0 : 0[0] -> 1[1] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213683:213683 [2] NCCL INFO Channel 20/0 : 2[2] -> 3[3] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213682:213682 [1] NCCL INFO Channel 21/0 : 1[1] -> 2[2] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213686:213686 [5] NCCL INFO Channel 17/0 : 5[5] -> 6[6] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213683:213683 [2] NCCL INFO Channel 21/0 : 2[2] -> 3[3] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213687:213687 [6] NCCL INFO Channel 18/0 : 6[6] -> 7[7] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213681:213681 [0] NCCL INFO Channel 14/0 : 0[0] -> 1[1] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213684:213684 [3] NCCL INFO Channel 21/0 : 3[3] -> 4[4] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213681:213681 [0] NCCL INFO Channel 15/0 : 0[0] -> 1[1] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213681:213681 [0] NCCL INFO Channel 16/0 : 0[0] -> 1[1] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213685:213685 [4] NCCL INFO Channel 17/0 : 4[4] -> 5[5] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213682:213682 [1] NCCL INFO Channel 22/0 : 1[1] -> 2[2] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213687:213687 [6] NCCL INFO Channel 19/0 : 6[6] -> 7[7] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213688:213688 [7] NCCL INFO Channel 21/0 : 7[7] -> 0[0] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213684:213684 [3] NCCL INFO Channel 22/0 : 3[3] -> 4[4] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213683:213683 [2] NCCL INFO Channel 22/0 : 2[2] -> 3[3] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213686:213686 [5] NCCL INFO Channel 18/0 : 5[5] -> 6[6] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213688:213688 [7] NCCL INFO Channel 22/0 : 7[7] -> 0[0] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213681:213681 [0] NCCL INFO Channel 17/0 : 0[0] -> 1[1] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213682:213682 [1] NCCL INFO Channel 23/0 : 1[1] -> 2[2] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213687:213687 [6] NCCL INFO Channel 20/0 : 6[6] -> 7[7] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213684:213684 [3] NCCL INFO Channel 23/0 : 3[3] -> 4[4] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213688:213688 [7] NCCL INFO Channel 23/0 : 7[7] -> 0[0] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213683:213683 [2] NCCL INFO Channel 23/0 : 2[2] -> 3[3] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213686:213686 [5] NCCL INFO Channel 19/0 : 5[5] -> 6[6] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213685:213685 [4] NCCL INFO Channel 18/0 : 4[4] -> 5[5] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213687:213687 [6] NCCL INFO Channel 21/0 : 6[6] -> 7[7] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213681:213681 [0] NCCL INFO Channel 18/0 : 0[0] -> 1[1] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213685:213685 [4] NCCL INFO Channel 19/0 : 4[4] -> 5[5] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213686:213686 [5] NCCL INFO Channel 20/0 : 5[5] -> 6[6] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213686:213686 [5] NCCL INFO Channel 21/0 : 5[5] -> 6[6] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213681:213681 [0] NCCL INFO Channel 19/0 : 0[0] -> 1[1] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213687:213687 [6] NCCL INFO Channel 22/0 : 6[6] -> 7[7] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213685:213685 [4] NCCL INFO Channel 20/0 : 4[4] -> 5[5] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213686:213686 [5] NCCL INFO Channel 22/0 : 5[5] -> 6[6] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213681:213681 [0] NCCL INFO Channel 20/0 : 0[0] -> 1[1] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213685:213685 [4] NCCL INFO Channel 21/0 : 4[4] -> 5[5] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213687:213687 [6] NCCL INFO Channel 23/0 : 6[6] -> 7[7] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213686:213686 [5] NCCL INFO Channel 23/0 : 5[5] -> 6[6] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213681:213681 [0] NCCL INFO Channel 21/0 : 0[0] -> 1[1] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213685:213685 [4] NCCL INFO Channel 22/0 : 4[4] -> 5[5] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213681:213681 [0] NCCL INFO Channel 22/0 : 0[0] -> 1[1] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213685:213685 [4] NCCL INFO Channel 23/0 : 4[4] -> 5[5] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213681:213681 [0] NCCL INFO Channel 23/0 : 0[0] -> 1[1] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:213686:213686 [5] NCCL INFO Connected all rings, use ring PXN 0 GDR 1 +ywang29-p4d-debug-1-worker-0:213687:213687 [6] NCCL INFO Connected all rings, use ring PXN 0 GDR 1 +ywang29-p4d-debug-1-worker-0:213682:213682 [1] NCCL INFO Connected all rings, use ring PXN 0 GDR 1 +ywang29-p4d-debug-1-worker-0:213684:213684 [3] NCCL INFO Connected all rings, use ring PXN 0 GDR 1 +ywang29-p4d-debug-1-worker-0:213688:213688 [7] NCCL INFO Connected all rings, use ring PXN 0 GDR 1 +ywang29-p4d-debug-1-worker-0:213681:213681 [0] NCCL INFO Connected all rings, use ring PXN 0 GDR 1 +ywang29-p4d-debug-1-worker-0:213685:213685 [4] NCCL INFO Connected all rings, use ring PXN 0 GDR 1 +ywang29-p4d-debug-1-worker-0:213683:213683 [2] NCCL INFO Connected all rings, use ring PXN 0 GDR 1 +Before initializing optimizer states +MA 4.73 GB Max_MA 4.73 GB CA 4.77 GB Max_CA 5 GB +CPU Virtual Memory: used = 66.75 GB, percent = 6.0% +After initializing optimizer states +MA 4.73 GB Max_MA 5.68 GB CA 5.71 GB Max_CA 6 GB +CPU Virtual Memory: used = 66.76 GB, percent = 6.0% +After initializing ZeRO optimizer +MA 4.73 GB Max_MA 4.73 GB CA 5.71 GB Max_CA 6 GB +CPU Virtual Memory: used = 66.76 GB, percent = 6.0% +[INFO|trainer.py:1587] 2026-04-29 15:39:23,284 >> ***** Running training ***** +[INFO|trainer.py:1588] 2026-04-29 15:39:23,284 >> Num examples = 30,000 +[INFO|trainer.py:1589] 2026-04-29 15:39:23,284 >> Num Epochs = 2 +[INFO|trainer.py:1590] 2026-04-29 15:39:23,284 >> Instantaneous batch size per device = 2 +[INFO|trainer.py:1593] 2026-04-29 15:39:23,284 >> Total train batch size (w. parallel, distributed & accumulation) = 128 +[INFO|trainer.py:1594] 2026-04-29 15:39:23,284 >> Gradient Accumulation steps = 8 +[INFO|trainer.py:1595] 2026-04-29 15:39:23,284 >> Total optimization steps = 470 +[INFO|trainer.py:1596] 2026-04-29 15:39:23,285 >> Number of trainable parameters = 2,031,739,904 +wandb: [wandb.login()] Loaded credentials for https://api.wandb.ai from WANDB_API_KEY. +wandb: Currently logged in as: kkhya (maskmoe) to https://api.wandb.ai. Use `wandb login --relogin` to force relogin +wandb: setting up run igs7r887 +wandb: Tracking run with wandb version 0.26.1 +wandb: Run data is saved locally in /nfs/ywang29/lm-factory/wandb/run-20260429_153923-igs7r887 +wandb: Run `wandb offline` to turn off syncing. +wandb: Syncing run qwen3_1.7b_coding_fft +wandb: ⭐️ View project at https://wandb.ai/maskmoe/MFT-LM +wandb: 🚀 View run at https://wandb.ai/maskmoe/MFT-LM/runs/igs7r887 + 0%| | 0/470 [00:00> Initializing 8 distributed tasks at: 127.0.0.1:39661 +W0429 15:40:08.187000 217197 site-packages/torch/distributed/run.py:852] +W0429 15:40:08.187000 217197 site-packages/torch/distributed/run.py:852] ***************************************** +W0429 15:40:08.187000 217197 site-packages/torch/distributed/run.py:852] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +W0429 15:40:08.187000 217197 site-packages/torch/distributed/run.py:852] ***************************************** +warmup_ratio is deprecated and will be removed in v5.2. Use `warmup_steps` instead. +warmup_ratio is deprecated and will be removed in v5.2. Use `warmup_steps` instead. +warmup_ratio is deprecated and will be removed in v5.2. Use `warmup_steps` instead. +warmup_ratio is deprecated and will be removed in v5.2. Use `warmup_steps` instead. +warmup_ratio is deprecated and will be removed in v5.2. Use `warmup_steps` instead. +warmup_ratio is deprecated and will be removed in v5.2. Use `warmup_steps` instead. +warmup_ratio is deprecated and will be removed in v5.2. Use `warmup_steps` instead. +warmup_ratio is deprecated and will be removed in v5.2. Use `warmup_steps` instead. +[W429 15:40:19.549342823 ProcessGroupNCCL.cpp:929] Warning: TORCH_NCCL_AVOID_RECORD_STREAMS is the default now, this environment variable is thus deprecated. (function operator()) +ywang29-p4d-debug-1-worker-0:217268:217268 [0] NCCL INFO ENV/Plugin: Could not find: libnccl-env.so +ywang29-p4d-debug-1-worker-0:217268:217268 [0] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth +ywang29-p4d-debug-1-worker-0:217268:217268 [0] NCCL INFO Bootstrap: Using eth0:10.200.170.96<0> +ywang29-p4d-debug-1-worker-0:217268:217268 [0] NCCL INFO cudaDriverVersion 13000 +ywang29-p4d-debug-1-worker-0:217268:217268 [0] NCCL INFO NCCL version 2.28.9+cuda13.0 +[W429 15:40:19.582759865 ProcessGroupNCCL.cpp:929] Warning: TORCH_NCCL_AVOID_RECORD_STREAMS is the default now, this environment variable is thus deprecated. (function operator()) +[W429 15:40:19.603828937 ProcessGroupNCCL.cpp:929] Warning: TORCH_NCCL_AVOID_RECORD_STREAMS is the default now, this environment variable is thus deprecated. (function operator()) +[W429 15:40:19.607721094 ProcessGroupNCCL.cpp:929] Warning: TORCH_NCCL_AVOID_RECORD_STREAMS is the default now, this environment variable is thus deprecated. (function operator()) +[W429 15:40:20.657556455 ProcessGroupNCCL.cpp:929] Warning: TORCH_NCCL_AVOID_RECORD_STREAMS is the default now, this environment variable is thus deprecated. (function operator()) +[W429 15:40:20.685145066 ProcessGroupNCCL.cpp:929] Warning: TORCH_NCCL_AVOID_RECORD_STREAMS is the default now, this environment variable is thus deprecated. (function operator()) +[W429 15:40:20.687763788 ProcessGroupNCCL.cpp:929] Warning: TORCH_NCCL_AVOID_RECORD_STREAMS is the default now, this environment variable is thus deprecated. (function operator()) +[W429 15:40:20.745175467 ProcessGroupNCCL.cpp:929] Warning: TORCH_NCCL_AVOID_RECORD_STREAMS is the default now, this environment variable is thus deprecated. (function operator()) +ywang29-p4d-debug-1-worker-0:217268:217268 [0] NCCL INFO Comm config Blocking set to 1 +ywang29-p4d-debug-1-worker-0:217279:217279 [7] NCCL INFO ENV/Plugin: Could not find: libnccl-env.so +ywang29-p4d-debug-1-worker-0:217279:217279 [7] NCCL INFO cudaDriverVersion 13000 +ywang29-p4d-debug-1-worker-0:217279:217279 [7] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth +ywang29-p4d-debug-1-worker-0:217279:217279 [7] NCCL INFO Bootstrap: Using eth0:10.200.170.96<0> +ywang29-p4d-debug-1-worker-0:217279:217279 [7] NCCL INFO NCCL version 2.28.9+cuda13.0 +ywang29-p4d-debug-1-worker-0:217279:217279 [7] NCCL INFO Comm config Blocking set to 1 +ywang29-p4d-debug-1-worker-0:217278:217278 [6] NCCL INFO ENV/Plugin: Could not find: libnccl-env.so +ywang29-p4d-debug-1-worker-0:217278:217278 [6] NCCL INFO cudaDriverVersion 13000 +ywang29-p4d-debug-1-worker-0:217278:217278 [6] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth +ywang29-p4d-debug-1-worker-0:217278:217278 [6] NCCL INFO Bootstrap: Using eth0:10.200.170.96<0> +ywang29-p4d-debug-1-worker-0:217278:217278 [6] NCCL INFO NCCL version 2.28.9+cuda13.0 +ywang29-p4d-debug-1-worker-0:217278:217278 [6] NCCL INFO Comm config Blocking set to 1 +ywang29-p4d-debug-1-worker-0:217274:217274 [4] NCCL INFO ENV/Plugin: Could not find: libnccl-env.so +ywang29-p4d-debug-1-worker-0:217274:217274 [4] NCCL INFO cudaDriverVersion 13000 +ywang29-p4d-debug-1-worker-0:217274:217274 [4] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth +ywang29-p4d-debug-1-worker-0:217274:217274 [4] NCCL INFO Bootstrap: Using eth0:10.200.170.96<0> +ywang29-p4d-debug-1-worker-0:217274:217274 [4] NCCL INFO NCCL version 2.28.9+cuda13.0 +ywang29-p4d-debug-1-worker-0:217274:217274 [4] NCCL INFO Comm config Blocking set to 1 +ywang29-p4d-debug-1-worker-0:217270:217270 [2] NCCL INFO ENV/Plugin: Could not find: libnccl-env.so +ywang29-p4d-debug-1-worker-0:217270:217270 [2] NCCL INFO cudaDriverVersion 13000 +ywang29-p4d-debug-1-worker-0:217270:217270 [2] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth +ywang29-p4d-debug-1-worker-0:217272:217272 [3] NCCL INFO ENV/Plugin: Could not find: libnccl-env.so +ywang29-p4d-debug-1-worker-0:217272:217272 [3] NCCL INFO cudaDriverVersion 13000 +ywang29-p4d-debug-1-worker-0:217270:217270 [2] NCCL INFO Bootstrap: Using eth0:10.200.170.96<0> +ywang29-p4d-debug-1-worker-0:217270:217270 [2] NCCL INFO NCCL version 2.28.9+cuda13.0 +ywang29-p4d-debug-1-worker-0:217275:217275 [5] NCCL INFO ENV/Plugin: Could not find: libnccl-env.so +ywang29-p4d-debug-1-worker-0:217275:217275 [5] NCCL INFO cudaDriverVersion 13000 +ywang29-p4d-debug-1-worker-0:217275:217275 [5] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth +ywang29-p4d-debug-1-worker-0:217272:217272 [3] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth +ywang29-p4d-debug-1-worker-0:217275:217275 [5] NCCL INFO Bootstrap: Using eth0:10.200.170.96<0> +ywang29-p4d-debug-1-worker-0:217275:217275 [5] NCCL INFO NCCL version 2.28.9+cuda13.0 +ywang29-p4d-debug-1-worker-0:217270:217270 [2] NCCL INFO Comm config Blocking set to 1 +ywang29-p4d-debug-1-worker-0:217272:217272 [3] NCCL INFO Bootstrap: Using eth0:10.200.170.96<0> +ywang29-p4d-debug-1-worker-0:217272:217272 [3] NCCL INFO NCCL version 2.28.9+cuda13.0 +ywang29-p4d-debug-1-worker-0:217275:217275 [5] NCCL INFO Comm config Blocking set to 1 +ywang29-p4d-debug-1-worker-0:217272:217272 [3] NCCL INFO Comm config Blocking set to 1 +ywang29-p4d-debug-1-worker-0:217269:217269 [1] NCCL INFO ENV/Plugin: Could not find: libnccl-env.so +ywang29-p4d-debug-1-worker-0:217269:217269 [1] NCCL INFO cudaDriverVersion 13000 +ywang29-p4d-debug-1-worker-0:217269:217269 [1] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth +ywang29-p4d-debug-1-worker-0:217269:217269 [1] NCCL INFO Bootstrap: Using eth0:10.200.170.96<0> +ywang29-p4d-debug-1-worker-0:217269:217269 [1] NCCL INFO NCCL version 2.28.9+cuda13.0 +ywang29-p4d-debug-1-worker-0:217269:217269 [1] NCCL INFO Comm config Blocking set to 1 +ywang29-p4d-debug-1-worker-0:217268:217268 [0] NCCL INFO NET/Plugin: Loaded net plugin Libfabric (v10) +ywang29-p4d-debug-1-worker-0:217268:217268 [0] NCCL INFO Successfully loaded external network plugin /opt/amazon/ofi-nccl/lib/libnccl-net.so +ywang29-p4d-debug-1-worker-0:217268:217268 [0] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth +ywang29-p4d-debug-1-worker-0:217268:217268 [0] NCCL INFO NET/Socket : Using [0]eth0:10.200.170.96<0> +ywang29-p4d-debug-1-worker-0:217268:217268 [0] NCCL INFO Initialized NET plugin Socket +ywang29-p4d-debug-1-worker-0:217268:217268 [0] NCCL INFO Assigned NET plugin Socket to comm +ywang29-p4d-debug-1-worker-0:217268:217268 [0] NCCL INFO Using network Socket +ywang29-p4d-debug-1-worker-0:217268:217268 [0] NCCL INFO ncclCommInitRankConfig comm 0x562c18cf2a40 rank 0 nranks 8 cudaDev 0 nvmlDev 0 busId 101c0 commId 0xf134627d563edc7d - Init START +ywang29-p4d-debug-1-worker-0:217278:217278 [6] NCCL INFO NET/Plugin: Loaded net plugin Libfabric (v10) +ywang29-p4d-debug-1-worker-0:217278:217278 [6] NCCL INFO Successfully loaded external network plugin /opt/amazon/ofi-nccl/lib/libnccl-net.so +ywang29-p4d-debug-1-worker-0:217278:217278 [6] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth +ywang29-p4d-debug-1-worker-0:217278:217278 [6] NCCL INFO NET/Socket : Using [0]eth0:10.200.170.96<0> +ywang29-p4d-debug-1-worker-0:217278:217278 [6] NCCL INFO Initialized NET plugin Socket +ywang29-p4d-debug-1-worker-0:217278:217278 [6] NCCL INFO Assigned NET plugin Socket to comm +ywang29-p4d-debug-1-worker-0:217278:217278 [6] NCCL INFO Using network Socket +ywang29-p4d-debug-1-worker-0:217274:217274 [4] NCCL INFO NET/Plugin: Loaded net plugin Libfabric (v10) +ywang29-p4d-debug-1-worker-0:217274:217274 [4] NCCL INFO Successfully loaded external network plugin /opt/amazon/ofi-nccl/lib/libnccl-net.so +ywang29-p4d-debug-1-worker-0:217274:217274 [4] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth +ywang29-p4d-debug-1-worker-0:217274:217274 [4] NCCL INFO NET/Socket : Using [0]eth0:10.200.170.96<0> +ywang29-p4d-debug-1-worker-0:217274:217274 [4] NCCL INFO Initialized NET plugin Socket +ywang29-p4d-debug-1-worker-0:217274:217274 [4] NCCL INFO Assigned NET plugin Socket to comm +ywang29-p4d-debug-1-worker-0:217274:217274 [4] NCCL INFO Using network Socket +ywang29-p4d-debug-1-worker-0:217278:217278 [6] NCCL INFO ncclCommInitRankConfig comm 0x56514433a870 rank 6 nranks 8 cudaDev 6 nvmlDev 6 busId a01c0 commId 0xf134627d563edc7d - Init START +ywang29-p4d-debug-1-worker-0:217270:217270 [2] NCCL INFO NET/Plugin: Loaded net plugin Libfabric (v10) +ywang29-p4d-debug-1-worker-0:217270:217270 [2] NCCL INFO Successfully loaded external network plugin /opt/amazon/ofi-nccl/lib/libnccl-net.so +ywang29-p4d-debug-1-worker-0:217279:217279 [7] NCCL INFO NET/Plugin: Loaded net plugin Libfabric (v10) +ywang29-p4d-debug-1-worker-0:217279:217279 [7] NCCL INFO Successfully loaded external network plugin /opt/amazon/ofi-nccl/lib/libnccl-net.so +ywang29-p4d-debug-1-worker-0:217270:217270 [2] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth +ywang29-p4d-debug-1-worker-0:217279:217279 [7] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth +ywang29-p4d-debug-1-worker-0:217279:217279 [7] NCCL INFO NET/Socket : Using [0]eth0:10.200.170.96<0> +ywang29-p4d-debug-1-worker-0:217270:217270 [2] NCCL INFO NET/Socket : Using [0]eth0:10.200.170.96<0> +ywang29-p4d-debug-1-worker-0:217270:217270 [2] NCCL INFO Initialized NET plugin Socket +ywang29-p4d-debug-1-worker-0:217279:217279 [7] NCCL INFO Initialized NET plugin Socket +ywang29-p4d-debug-1-worker-0:217270:217270 [2] NCCL INFO Assigned NET plugin Socket to comm +ywang29-p4d-debug-1-worker-0:217279:217279 [7] NCCL INFO Assigned NET plugin Socket to comm +ywang29-p4d-debug-1-worker-0:217270:217270 [2] NCCL INFO Using network Socket +ywang29-p4d-debug-1-worker-0:217279:217279 [7] NCCL INFO Using network Socket +ywang29-p4d-debug-1-worker-0:217272:217272 [3] NCCL INFO NET/Plugin: Loaded net plugin Libfabric (v10) +ywang29-p4d-debug-1-worker-0:217275:217275 [5] NCCL INFO NET/Plugin: Loaded net plugin Libfabric (v10) +ywang29-p4d-debug-1-worker-0:217272:217272 [3] NCCL INFO Successfully loaded external network plugin /opt/amazon/ofi-nccl/lib/libnccl-net.so +ywang29-p4d-debug-1-worker-0:217275:217275 [5] NCCL INFO Successfully loaded external network plugin /opt/amazon/ofi-nccl/lib/libnccl-net.so +ywang29-p4d-debug-1-worker-0:217275:217275 [5] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth +ywang29-p4d-debug-1-worker-0:217272:217272 [3] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth +ywang29-p4d-debug-1-worker-0:217275:217275 [5] NCCL INFO NET/Socket : Using [0]eth0:10.200.170.96<0> +ywang29-p4d-debug-1-worker-0:217272:217272 [3] NCCL INFO NET/Socket : Using [0]eth0:10.200.170.96<0> +ywang29-p4d-debug-1-worker-0:217272:217272 [3] NCCL INFO Initialized NET plugin Socket +ywang29-p4d-debug-1-worker-0:217275:217275 [5] NCCL INFO Initialized NET plugin Socket +ywang29-p4d-debug-1-worker-0:217272:217272 [3] NCCL INFO Assigned NET plugin Socket to comm +ywang29-p4d-debug-1-worker-0:217272:217272 [3] NCCL INFO Using network Socket +ywang29-p4d-debug-1-worker-0:217275:217275 [5] NCCL INFO Assigned NET plugin Socket to comm +ywang29-p4d-debug-1-worker-0:217275:217275 [5] NCCL INFO Using network Socket +ywang29-p4d-debug-1-worker-0:217274:217274 [4] NCCL INFO ncclCommInitRankConfig comm 0x56199da84a70 rank 4 nranks 8 cudaDev 4 nvmlDev 4 busId 901c0 commId 0xf134627d563edc7d - Init START +ywang29-p4d-debug-1-worker-0:217269:217269 [1] NCCL INFO NET/Plugin: Loaded net plugin Libfabric (v10) +ywang29-p4d-debug-1-worker-0:217269:217269 [1] NCCL INFO Successfully loaded external network plugin /opt/amazon/ofi-nccl/lib/libnccl-net.so +ywang29-p4d-debug-1-worker-0:217269:217269 [1] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth +ywang29-p4d-debug-1-worker-0:217269:217269 [1] NCCL INFO NET/Socket : Using [0]eth0:10.200.170.96<0> +ywang29-p4d-debug-1-worker-0:217269:217269 [1] NCCL INFO Initialized NET plugin Socket +ywang29-p4d-debug-1-worker-0:217269:217269 [1] NCCL INFO Assigned NET plugin Socket to comm +ywang29-p4d-debug-1-worker-0:217269:217269 [1] NCCL INFO Using network Socket +ywang29-p4d-debug-1-worker-0:217272:217272 [3] NCCL INFO ncclCommInitRankConfig comm 0x557edc8b65e0 rank 3 nranks 8 cudaDev 3 nvmlDev 3 busId 201d0 commId 0xf134627d563edc7d - Init START +ywang29-p4d-debug-1-worker-0:217275:217275 [5] NCCL INFO ncclCommInitRankConfig comm 0x55c53c3343a0 rank 5 nranks 8 cudaDev 5 nvmlDev 5 busId 901d0 commId 0xf134627d563edc7d - Init START +ywang29-p4d-debug-1-worker-0:217279:217279 [7] NCCL INFO ncclCommInitRankConfig comm 0x5556d7965790 rank 7 nranks 8 cudaDev 7 nvmlDev 7 busId a01d0 commId 0xf134627d563edc7d - Init START +ywang29-p4d-debug-1-worker-0:217270:217270 [2] NCCL INFO ncclCommInitRankConfig comm 0x55feadedf680 rank 2 nranks 8 cudaDev 2 nvmlDev 2 busId 201c0 commId 0xf134627d563edc7d - Init START +ywang29-p4d-debug-1-worker-0:217275:217275 [5] NCCL INFO RAS client listening socket at ::1<28028> +ywang29-p4d-debug-1-worker-0:217274:217274 [4] NCCL INFO RAS client listening socket at ::1<28028> +ywang29-p4d-debug-1-worker-0:217278:217278 [6] NCCL INFO RAS client listening socket at ::1<28028> +ywang29-p4d-debug-1-worker-0:217279:217279 [7] NCCL INFO RAS client listening socket at ::1<28028> +ywang29-p4d-debug-1-worker-0:217272:217272 [3] NCCL INFO RAS client listening socket at ::1<28028> +ywang29-p4d-debug-1-worker-0:217269:217269 [1] NCCL INFO ncclCommInitRankConfig comm 0x55ef466ff050 rank 1 nranks 8 cudaDev 1 nvmlDev 1 busId 101d0 commId 0xf134627d563edc7d - Init START +ywang29-p4d-debug-1-worker-0:217269:217269 [1] NCCL INFO RAS client listening socket at ::1<28028> +ywang29-p4d-debug-1-worker-0:217270:217270 [2] NCCL INFO RAS client listening socket at ::1<28028> +ywang29-p4d-debug-1-worker-0:217268:217268 [0] NCCL INFO RAS client listening socket at ::1<28028> +ywang29-p4d-debug-1-worker-0:217268:217268 [0] NCCL INFO Bootstrap timings total 0.139276 (create 0.000055, send 0.000162, recv 0.138538, ring 0.000111, delay 0.000001) +ywang29-p4d-debug-1-worker-0:217269:217269 [1] NCCL INFO Bootstrap timings total 0.000810 (create 0.000036, send 0.000068, recv 0.000221, ring 0.000121, delay 0.000001) +ywang29-p4d-debug-1-worker-0:217279:217279 [7] NCCL INFO Bootstrap timings total 0.015638 (create 0.000045, send 0.000091, recv 0.000314, ring 0.014702, delay 0.000001) +ywang29-p4d-debug-1-worker-0:217278:217278 [6] NCCL INFO Bootstrap timings total 0.100182 (create 0.000043, send 0.000073, recv 0.084834, ring 0.014758, delay 0.000001) +ywang29-p4d-debug-1-worker-0:217274:217274 [4] NCCL INFO Bootstrap timings total 0.089890 (create 0.000058, send 0.000072, recv 0.074523, ring 0.014770, delay 0.000001) +ywang29-p4d-debug-1-worker-0:217270:217270 [2] NCCL INFO Bootstrap timings total 0.014903 (create 0.000047, send 0.000074, recv 0.000064, ring 0.000116, delay 0.000001) +ywang29-p4d-debug-1-worker-0:217275:217275 [5] NCCL INFO Bootstrap timings total 0.016054 (create 0.000035, send 0.000064, recv 0.000281, ring 0.014771, delay 0.000001) +ywang29-p4d-debug-1-worker-0:217272:217272 [3] NCCL INFO Bootstrap timings total 0.018770 (create 0.000043, send 0.000073, recv 0.000174, ring 0.014344, delay 0.000001) +ywang29-p4d-debug-1-worker-0:217275:217275 [5] NCCL INFO ncclTopoGetCpuAffinity: Affinity for GPU 5 is 24-47,72-95. (GPU affinity = 24-47,72-95 ; CPU affinity = 0-95). +ywang29-p4d-debug-1-worker-0:217275:217275 [5] NCCL INFO NVLS multicast support is not available on dev 5 (NVLS_NCHANNELS 0) +ywang29-p4d-debug-1-worker-0:217274:217274 [4] NCCL INFO ncclTopoGetCpuAffinity: Affinity for GPU 4 is 24-47,72-95. (GPU affinity = 24-47,72-95 ; CPU affinity = 0-95). +ywang29-p4d-debug-1-worker-0:217274:217274 [4] NCCL INFO NVLS multicast support is not available on dev 4 (NVLS_NCHANNELS 0) +ywang29-p4d-debug-1-worker-0:217278:217278 [6] NCCL INFO ncclTopoGetCpuAffinity: Affinity for GPU 6 is 24-47,72-95. (GPU affinity = 24-47,72-95 ; CPU affinity = 0-95). +ywang29-p4d-debug-1-worker-0:217272:217272 [3] NCCL INFO ncclTopoGetCpuAffinity: Affinity for GPU 3 is 0-23,48-71. (GPU affinity = 0-23,48-71 ; CPU affinity = 0-95). +ywang29-p4d-debug-1-worker-0:217272:217272 [3] NCCL INFO NVLS multicast support is not available on dev 3 (NVLS_NCHANNELS 0) +ywang29-p4d-debug-1-worker-0:217278:217278 [6] NCCL INFO NVLS multicast support is not available on dev 6 (NVLS_NCHANNELS 0) +ywang29-p4d-debug-1-worker-0:217279:217279 [7] NCCL INFO ncclTopoGetCpuAffinity: Affinity for GPU 7 is 24-47,72-95. (GPU affinity = 24-47,72-95 ; CPU affinity = 0-95). +ywang29-p4d-debug-1-worker-0:217279:217279 [7] NCCL INFO NVLS multicast support is not available on dev 7 (NVLS_NCHANNELS 0) +ywang29-p4d-debug-1-worker-0:217270:217270 [2] NCCL INFO ncclTopoGetCpuAffinity: Affinity for GPU 2 is 0-23,48-71. (GPU affinity = 0-23,48-71 ; CPU affinity = 0-95). +ywang29-p4d-debug-1-worker-0:217270:217270 [2] NCCL INFO NVLS multicast support is not available on dev 2 (NVLS_NCHANNELS 0) +ywang29-p4d-debug-1-worker-0:217269:217269 [1] NCCL INFO ncclTopoGetCpuAffinity: Affinity for GPU 1 is 0-23,48-71. (GPU affinity = 0-23,48-71 ; CPU affinity = 0-95). +ywang29-p4d-debug-1-worker-0:217268:217268 [0] NCCL INFO ncclTopoGetCpuAffinity: Affinity for GPU 0 is 0-23,48-71. (GPU affinity = 0-23,48-71 ; CPU affinity = 0-95). +ywang29-p4d-debug-1-worker-0:217268:217268 [0] NCCL INFO NVLS multicast support is not available on dev 0 (NVLS_NCHANNELS 0) +ywang29-p4d-debug-1-worker-0:217269:217269 [1] NCCL INFO NVLS multicast support is not available on dev 1 (NVLS_NCHANNELS 0) +ywang29-p4d-debug-1-worker-0:217270:217270 [2] NCCL INFO comm 0x55feadedf680 rank 2 nRanks 8 nNodes 1 localRanks 8 localRank 2 MNNVL 0 +ywang29-p4d-debug-1-worker-0:217275:217275 [5] NCCL INFO comm 0x55c53c3343a0 rank 5 nRanks 8 nNodes 1 localRanks 8 localRank 5 MNNVL 0 +ywang29-p4d-debug-1-worker-0:217269:217269 [1] NCCL INFO comm 0x55ef466ff050 rank 1 nRanks 8 nNodes 1 localRanks 8 localRank 1 MNNVL 0 +ywang29-p4d-debug-1-worker-0:217272:217272 [3] NCCL INFO comm 0x557edc8b65e0 rank 3 nRanks 8 nNodes 1 localRanks 8 localRank 3 MNNVL 0 +ywang29-p4d-debug-1-worker-0:217278:217278 [6] NCCL INFO comm 0x56514433a870 rank 6 nRanks 8 nNodes 1 localRanks 8 localRank 6 MNNVL 0 +ywang29-p4d-debug-1-worker-0:217274:217274 [4] NCCL INFO comm 0x56199da84a70 rank 4 nRanks 8 nNodes 1 localRanks 8 localRank 4 MNNVL 0 +ywang29-p4d-debug-1-worker-0:217268:217268 [0] NCCL INFO comm 0x562c18cf2a40 rank 0 nRanks 8 nNodes 1 localRanks 8 localRank 0 MNNVL 0 +ywang29-p4d-debug-1-worker-0:217279:217279 [7] NCCL INFO comm 0x5556d7965790 rank 7 nRanks 8 nNodes 1 localRanks 8 localRank 7 MNNVL 0 +ywang29-p4d-debug-1-worker-0:217268:217268 [0] NCCL INFO Channel 00/24 : 0 1 2 3 4 5 6 7 +ywang29-p4d-debug-1-worker-0:217268:217268 [0] NCCL INFO Channel 01/24 : 0 1 2 3 4 5 6 7 +ywang29-p4d-debug-1-worker-0:217268:217268 [0] NCCL INFO Channel 02/24 : 0 1 2 3 4 5 6 7 +ywang29-p4d-debug-1-worker-0:217268:217268 [0] NCCL INFO Channel 03/24 : 0 1 2 3 4 5 6 7 +ywang29-p4d-debug-1-worker-0:217270:217270 [2] NCCL INFO Trees [0] 3/-1/-1->2->1 [1] 3/-1/-1->2->1 [2] 3/-1/-1->2->1 [3] 3/-1/-1->2->1 [4] 3/-1/-1->2->1 [5] 3/-1/-1->2->1 [6] 3/-1/-1->2->1 [7] 3/-1/-1->2->1 [8] 3/-1/-1->2->1 [9] 3/-1/-1->2->1 [10] 3/-1/-1->2->1 [11] 3/-1/-1->2->1 [12] 3/-1/-1->2->1 [13] 3/-1/-1->2->1 [14] 3/-1/-1->2->1 [15] 3/-1/-1->2->1 [16] 3/-1/-1->2->1 [17] 3/-1/-1->2->1 [18] 3/-1/-1->2->1 [19] 3/-1/-1->2->1 [20] 3/-1/-1->2->1 [21] 3/-1/-1->2->1 [22] 3/-1/-1->2->1 [23] 3/-1/-1->2->1 +ywang29-p4d-debug-1-worker-0:217268:217268 [0] NCCL INFO Channel 04/24 : 0 1 2 3 4 5 6 7 +ywang29-p4d-debug-1-worker-0:217269:217269 [1] NCCL INFO Trees [0] 2/-1/-1->1->0 [1] 2/-1/-1->1->0 [2] 2/-1/-1->1->0 [3] 2/-1/-1->1->0 [4] 2/-1/-1->1->0 [5] 2/-1/-1->1->0 [6] 2/-1/-1->1->0 [7] 2/-1/-1->1->0 [8] 2/-1/-1->1->0 [9] 2/-1/-1->1->0 [10] 2/-1/-1->1->0 [11] 2/-1/-1->1->0 [12] 2/-1/-1->1->0 [13] 2/-1/-1->1->0 [14] 2/-1/-1->1->0 [15] 2/-1/-1->1->0 [16] 2/-1/-1->1->0 [17] 2/-1/-1->1->0 [18] 2/-1/-1->1->0 [19] 2/-1/-1->1->0 [20] 2/-1/-1->1->0 [21] 2/-1/-1->1->0 [22] 2/-1/-1->1->0 [23] 2/-1/-1->1->0 +ywang29-p4d-debug-1-worker-0:217268:217268 [0] NCCL INFO Channel 05/24 : 0 1 2 3 4 5 6 7 +ywang29-p4d-debug-1-worker-0:217270:217270 [2] NCCL INFO P2P Chunksize set to 524288 +ywang29-p4d-debug-1-worker-0:217268:217268 [0] NCCL INFO Channel 06/24 : 0 1 2 3 4 5 6 7 +ywang29-p4d-debug-1-worker-0:217272:217272 [3] NCCL INFO Trees [0] 4/-1/-1->3->2 [1] 4/-1/-1->3->2 [2] 4/-1/-1->3->2 [3] 4/-1/-1->3->2 [4] 4/-1/-1->3->2 [5] 4/-1/-1->3->2 [6] 4/-1/-1->3->2 [7] 4/-1/-1->3->2 [8] 4/-1/-1->3->2 [9] 4/-1/-1->3->2 [10] 4/-1/-1->3->2 [11] 4/-1/-1->3->2 [12] 4/-1/-1->3->2 [13] 4/-1/-1->3->2 [14] 4/-1/-1->3->2 [15] 4/-1/-1->3->2 [16] 4/-1/-1->3->2 [17] 4/-1/-1->3->2 [18] 4/-1/-1->3->2 [19] 4/-1/-1->3->2 [20] 4/-1/-1->3->2 [21] 4/-1/-1->3->2 [22] 4/-1/-1->3->2 [23] 4/-1/-1->3->2 +ywang29-p4d-debug-1-worker-0:217275:217275 [5] NCCL INFO Trees [0] 6/-1/-1->5->4 [1] 6/-1/-1->5->4 [2] 6/-1/-1->5->4 [3] 6/-1/-1->5->4 [4] 6/-1/-1->5->4 [5] 6/-1/-1->5->4 [6] 6/-1/-1->5->4 [7] 6/-1/-1->5->4 [8] 6/-1/-1->5->4 [9] 6/-1/-1->5->4 [10] 6/-1/-1->5->4 [11] 6/-1/-1->5->4 [12] 6/-1/-1->5->4 [13] 6/-1/-1->5->4 [14] 6/-1/-1->5->4 [15] 6/-1/-1->5->4 [16] 6/-1/-1->5->4 [17] 6/-1/-1->5->4 [18] 6/-1/-1->5->4 [19] 6/-1/-1->5->4 [20] 6/-1/-1->5->4 [21] 6/-1/-1->5->4 [22] 6/-1/-1->5->4 [23] 6/-1/-1->5->4 +ywang29-p4d-debug-1-worker-0:217274:217274 [4] NCCL INFO Trees [0] 5/-1/-1->4->3 [1] 5/-1/-1->4->3 [2] 5/-1/-1->4->3 [3] 5/-1/-1->4->3 [4] 5/-1/-1->4->3 [5] 5/-1/-1->4->3 [6] 5/-1/-1->4->3 [7] 5/-1/-1->4->3 [8] 5/-1/-1->4->3 [9] 5/-1/-1->4->3 [10] 5/-1/-1->4->3 [11] 5/-1/-1->4->3 [12] 5/-1/-1->4->3 [13] 5/-1/-1->4->3 [14] 5/-1/-1->4->3 [15] 5/-1/-1->4->3 [16] 5/-1/-1->4->3 [17] 5/-1/-1->4->3 [18] 5/-1/-1->4->3 [19] 5/-1/-1->4->3 [20] 5/-1/-1->4->3 [21] 5/-1/-1->4->3 [22] 5/-1/-1->4->3 [23] 5/-1/-1->4->3 +ywang29-p4d-debug-1-worker-0:217269:217269 [1] NCCL INFO P2P Chunksize set to 524288 +ywang29-p4d-debug-1-worker-0:217268:217268 [0] NCCL INFO Channel 07/24 : 0 1 2 3 4 5 6 7 +ywang29-p4d-debug-1-worker-0:217278:217278 [6] NCCL INFO Trees [0] 7/-1/-1->6->5 [1] 7/-1/-1->6->5 [2] 7/-1/-1->6->5 [3] 7/-1/-1->6->5 [4] 7/-1/-1->6->5 [5] 7/-1/-1->6->5 [6] 7/-1/-1->6->5 [7] 7/-1/-1->6->5 [8] 7/-1/-1->6->5 [9] 7/-1/-1->6->5 [10] 7/-1/-1->6->5 [11] 7/-1/-1->6->5 [12] 7/-1/-1->6->5 [13] 7/-1/-1->6->5 [14] 7/-1/-1->6->5 [15] 7/-1/-1->6->5 [16] 7/-1/-1->6->5 [17] 7/-1/-1->6->5 [18] 7/-1/-1->6->5 [19] 7/-1/-1->6->5 [20] 7/-1/-1->6->5 [21] 7/-1/-1->6->5 [22] 7/-1/-1->6->5 [23] 7/-1/-1->6->5 +ywang29-p4d-debug-1-worker-0:217268:217268 [0] NCCL INFO Channel 08/24 : 0 1 2 3 4 5 6 7 +ywang29-p4d-debug-1-worker-0:217272:217272 [3] NCCL INFO P2P Chunksize set to 524288 +ywang29-p4d-debug-1-worker-0:217275:217275 [5] NCCL INFO P2P Chunksize set to 524288 +ywang29-p4d-debug-1-worker-0:217274:217274 [4] NCCL INFO P2P Chunksize set to 524288 +ywang29-p4d-debug-1-worker-0:217279:217279 [7] NCCL INFO Trees [0] -1/-1/-1->7->6 [1] -1/-1/-1->7->6 [2] -1/-1/-1->7->6 [3] -1/-1/-1->7->6 [4] -1/-1/-1->7->6 [5] -1/-1/-1->7->6 [6] -1/-1/-1->7->6 [7] -1/-1/-1->7->6 [8] -1/-1/-1->7->6 [9] -1/-1/-1->7->6 [10] -1/-1/-1->7->6 [11] -1/-1/-1->7->6 [12] -1/-1/-1->7->6 [13] -1/-1/-1->7->6 [14] -1/-1/-1->7->6 [15] -1/-1/-1->7->6 [16] -1/-1/-1->7->6 [17] -1/-1/-1->7->6 [18] -1/-1/-1->7->6 [19] -1/-1/-1->7->6 [20] -1/-1/-1->7->6 [21] -1/-1/-1->7->6 [22] -1/-1/-1->7->6 [23] -1/-1/-1->7->6 +ywang29-p4d-debug-1-worker-0:217278:217278 [6] NCCL INFO P2P Chunksize set to 524288 +ywang29-p4d-debug-1-worker-0:217268:217268 [0] NCCL INFO Channel 09/24 : 0 1 2 3 4 5 6 7 +ywang29-p4d-debug-1-worker-0:217268:217268 [0] NCCL INFO Channel 10/24 : 0 1 2 3 4 5 6 7 +ywang29-p4d-debug-1-worker-0:217279:217279 [7] NCCL INFO P2P Chunksize set to 524288 +ywang29-p4d-debug-1-worker-0:217268:217268 [0] NCCL INFO Channel 11/24 : 0 1 2 3 4 5 6 7 +ywang29-p4d-debug-1-worker-0:217268:217268 [0] NCCL INFO Channel 12/24 : 0 1 2 3 4 5 6 7 +ywang29-p4d-debug-1-worker-0:217268:217268 [0] NCCL INFO Channel 13/24 : 0 1 2 3 4 5 6 7 +ywang29-p4d-debug-1-worker-0:217268:217268 [0] NCCL INFO Channel 14/24 : 0 1 2 3 4 5 6 7 +ywang29-p4d-debug-1-worker-0:217268:217268 [0] NCCL INFO Channel 15/24 : 0 1 2 3 4 5 6 7 +ywang29-p4d-debug-1-worker-0:217268:217268 [0] NCCL INFO Channel 16/24 : 0 1 2 3 4 5 6 7 +ywang29-p4d-debug-1-worker-0:217268:217268 [0] NCCL INFO Channel 17/24 : 0 1 2 3 4 5 6 7 +ywang29-p4d-debug-1-worker-0:217268:217268 [0] NCCL INFO Channel 18/24 : 0 1 2 3 4 5 6 7 +ywang29-p4d-debug-1-worker-0:217268:217268 [0] NCCL INFO Channel 19/24 : 0 1 2 3 4 5 6 7 +ywang29-p4d-debug-1-worker-0:217268:217268 [0] NCCL INFO Channel 20/24 : 0 1 2 3 4 5 6 7 +ywang29-p4d-debug-1-worker-0:217268:217268 [0] NCCL INFO Channel 21/24 : 0 1 2 3 4 5 6 7 +ywang29-p4d-debug-1-worker-0:217268:217268 [0] NCCL INFO Channel 22/24 : 0 1 2 3 4 5 6 7 +ywang29-p4d-debug-1-worker-0:217268:217268 [0] NCCL INFO Channel 23/24 : 0 1 2 3 4 5 6 7 +ywang29-p4d-debug-1-worker-0:217268:217268 [0] NCCL INFO Trees [0] 1/-1/-1->0->-1 [1] 1/-1/-1->0->-1 [2] 1/-1/-1->0->-1 [3] 1/-1/-1->0->-1 [4] 1/-1/-1->0->-1 [5] 1/-1/-1->0->-1 [6] 1/-1/-1->0->-1 [7] 1/-1/-1->0->-1 [8] 1/-1/-1->0->-1 [9] 1/-1/-1->0->-1 [10] 1/-1/-1->0->-1 [11] 1/-1/-1->0->-1 [12] 1/-1/-1->0->-1 [13] 1/-1/-1->0->-1 [14] 1/-1/-1->0->-1 [15] 1/-1/-1->0->-1 [16] 1/-1/-1->0->-1 [17] 1/-1/-1->0->-1 [18] 1/-1/-1->0->-1 [19] 1/-1/-1->0->-1 [20] 1/-1/-1->0->-1 [21] 1/-1/-1->0->-1 [22] 1/-1/-1->0->-1 [23] 1/-1/-1->0->-1 +ywang29-p4d-debug-1-worker-0:217268:217268 [0] NCCL INFO P2P Chunksize set to 524288 +ywang29-p4d-debug-1-worker-0:217274:217274 [4] NCCL INFO PROFILER/Plugin: Could not find: libnccl-profiler.so +ywang29-p4d-debug-1-worker-0:217274:217274 [4] NCCL INFO Check P2P Type isAllDirectP2p 1 directMode 0 isAllCudaP2p 1 +ywang29-p4d-debug-1-worker-0:217274:217684 [0] NCCL INFO [Proxy Service UDS] Device 4 CPU core 31 +ywang29-p4d-debug-1-worker-0:217274:217683 [0] NCCL INFO [Proxy Service] Device 4 CPU core 29 +ywang29-p4d-debug-1-worker-0:217270:217270 [2] NCCL INFO PROFILER/Plugin: Could not find: libnccl-profiler.so +ywang29-p4d-debug-1-worker-0:217270:217270 [2] NCCL INFO Check P2P Type isAllDirectP2p 1 directMode 0 isAllCudaP2p 1 +ywang29-p4d-debug-1-worker-0:217270:217685 [0] NCCL INFO [Proxy Service] Device 2 CPU core 67 +ywang29-p4d-debug-1-worker-0:217270:217686 [0] NCCL INFO [Proxy Service UDS] Device 2 CPU core 50 +ywang29-p4d-debug-1-worker-0:217278:217278 [6] NCCL INFO PROFILER/Plugin: Could not find: libnccl-profiler.so +ywang29-p4d-debug-1-worker-0:217278:217278 [6] NCCL INFO Check P2P Type isAllDirectP2p 1 directMode 0 isAllCudaP2p 1 +ywang29-p4d-debug-1-worker-0:217278:217687 [0] NCCL INFO [Proxy Service] Device 6 CPU core 38 +ywang29-p4d-debug-1-worker-0:217278:217688 [0] NCCL INFO [Proxy Service UDS] Device 6 CPU core 88 +ywang29-p4d-debug-1-worker-0:217279:217279 [7] NCCL INFO PROFILER/Plugin: Could not find: libnccl-profiler.so +ywang29-p4d-debug-1-worker-0:217269:217269 [1] NCCL INFO PROFILER/Plugin: Could not find: libnccl-profiler.so +ywang29-p4d-debug-1-worker-0:217279:217279 [7] NCCL INFO Check P2P Type isAllDirectP2p 1 directMode 0 isAllCudaP2p 1 +ywang29-p4d-debug-1-worker-0:217269:217269 [1] NCCL INFO Check P2P Type isAllDirectP2p 1 directMode 0 isAllCudaP2p 1 +ywang29-p4d-debug-1-worker-0:217269:217690 [0] NCCL INFO [Proxy Service] Device 1 CPU core 5 +ywang29-p4d-debug-1-worker-0:217279:217689 [0] NCCL INFO [Proxy Service] Device 7 CPU core 91 +ywang29-p4d-debug-1-worker-0:217269:217691 [0] NCCL INFO [Proxy Service UDS] Device 1 CPU core 54 +ywang29-p4d-debug-1-worker-0:217279:217692 [0] NCCL INFO [Proxy Service UDS] Device 7 CPU core 93 +ywang29-p4d-debug-1-worker-0:217272:217272 [3] NCCL INFO PROFILER/Plugin: Could not find: libnccl-profiler.so +ywang29-p4d-debug-1-worker-0:217272:217272 [3] NCCL INFO Check P2P Type isAllDirectP2p 1 directMode 0 isAllCudaP2p 1 +ywang29-p4d-debug-1-worker-0:217268:217268 [0] NCCL INFO PROFILER/Plugin: Could not find: libnccl-profiler.so +ywang29-p4d-debug-1-worker-0:217268:217268 [0] NCCL INFO Check P2P Type isAllDirectP2p 1 directMode 0 isAllCudaP2p 1 +ywang29-p4d-debug-1-worker-0:217272:217693 [0] NCCL INFO [Proxy Service] Device 3 CPU core 11 +ywang29-p4d-debug-1-worker-0:217272:217694 [0] NCCL INFO [Proxy Service UDS] Device 3 CPU core 22 +ywang29-p4d-debug-1-worker-0:217268:217696 [0] NCCL INFO [Proxy Service UDS] Device 0 CPU core 15 +ywang29-p4d-debug-1-worker-0:217268:217695 [0] NCCL INFO [Proxy Service] Device 0 CPU core 13 +ywang29-p4d-debug-1-worker-0:217275:217275 [5] NCCL INFO PROFILER/Plugin: Could not find: libnccl-profiler.so +ywang29-p4d-debug-1-worker-0:217275:217275 [5] NCCL INFO Check P2P Type isAllDirectP2p 1 directMode 0 isAllCudaP2p 1 +ywang29-p4d-debug-1-worker-0:217275:217697 [0] NCCL INFO [Proxy Service] Device 5 CPU core 94 +ywang29-p4d-debug-1-worker-0:217275:217698 [0] NCCL INFO [Proxy Service UDS] Device 5 CPU core 81 +ywang29-p4d-debug-1-worker-0:217274:217274 [4] NCCL INFO TUNER/Plugin: Could not find: libnccl-tuner.so +ywang29-p4d-debug-1-worker-0:217274:217274 [4] NCCL INFO TUNER/Plugin: Using nccl_ofi_tuner (v3) +ywang29-p4d-debug-1-worker-0:217274:217274 [4] NCCL INFO NET/OFI NCCL_OFI_TUNER is not available for platform : p4de.24xlarge, Fall back to NCCL's tuner +ywang29-p4d-debug-1-worker-0:217274:217274 [4] NCCL INFO threadThresholds 8/8/64 | 64/8/64 | 512 | 512 +ywang29-p4d-debug-1-worker-0:217274:217274 [4] NCCL INFO 24 coll channels, 24 collnet channels, 0 nvls channels, 32 p2p channels, 32 p2p channels per peer +ywang29-p4d-debug-1-worker-0:217275:217275 [5] NCCL INFO TUNER/Plugin: Could not find: libnccl-tuner.so +ywang29-p4d-debug-1-worker-0:217278:217278 [6] NCCL INFO TUNER/Plugin: Could not find: libnccl-tuner.so +ywang29-p4d-debug-1-worker-0:217278:217278 [6] NCCL INFO TUNER/Plugin: Using nccl_ofi_tuner (v3) +ywang29-p4d-debug-1-worker-0:217275:217275 [5] NCCL INFO TUNER/Plugin: Using nccl_ofi_tuner (v3) +ywang29-p4d-debug-1-worker-0:217278:217278 [6] NCCL INFO NET/OFI NCCL_OFI_TUNER is not available for platform : p4de.24xlarge, Fall back to NCCL's tuner +ywang29-p4d-debug-1-worker-0:217275:217275 [5] NCCL INFO NET/OFI NCCL_OFI_TUNER is not available for platform : p4de.24xlarge, Fall back to NCCL's tuner +ywang29-p4d-debug-1-worker-0:217278:217278 [6] NCCL INFO threadThresholds 8/8/64 | 64/8/64 | 512 | 512 +ywang29-p4d-debug-1-worker-0:217278:217278 [6] NCCL INFO 24 coll channels, 24 collnet channels, 0 nvls channels, 32 p2p channels, 32 p2p channels per peer +ywang29-p4d-debug-1-worker-0:217275:217275 [5] NCCL INFO threadThresholds 8/8/64 | 64/8/64 | 512 | 512 +ywang29-p4d-debug-1-worker-0:217275:217275 [5] NCCL INFO 24 coll channels, 24 collnet channels, 0 nvls channels, 32 p2p channels, 32 p2p channels per peer +ywang29-p4d-debug-1-worker-0:217268:217268 [0] NCCL INFO TUNER/Plugin: Could not find: libnccl-tuner.so +ywang29-p4d-debug-1-worker-0:217268:217268 [0] NCCL INFO TUNER/Plugin: Using nccl_ofi_tuner (v3) +ywang29-p4d-debug-1-worker-0:217268:217268 [0] NCCL INFO NET/OFI NCCL_OFI_TUNER is not available for platform : p4de.24xlarge, Fall back to NCCL's tuner +ywang29-p4d-debug-1-worker-0:217268:217268 [0] NCCL INFO threadThresholds 8/8/64 | 64/8/64 | 512 | 512 +ywang29-p4d-debug-1-worker-0:217268:217268 [0] NCCL INFO 24 coll channels, 24 collnet channels, 0 nvls channels, 32 p2p channels, 32 p2p channels per peer +ywang29-p4d-debug-1-worker-0:217268:217268 [0] NCCL INFO CC Off, workFifoBytes 1048576 +ywang29-p4d-debug-1-worker-0:217270:217270 [2] NCCL INFO TUNER/Plugin: Could not find: libnccl-tuner.so +ywang29-p4d-debug-1-worker-0:217270:217270 [2] NCCL INFO TUNER/Plugin: Using nccl_ofi_tuner (v3) +ywang29-p4d-debug-1-worker-0:217270:217270 [2] NCCL INFO NET/OFI NCCL_OFI_TUNER is not available for platform : p4de.24xlarge, Fall back to NCCL's tuner +ywang29-p4d-debug-1-worker-0:217270:217270 [2] NCCL INFO threadThresholds 8/8/64 | 64/8/64 | 512 | 512 +ywang29-p4d-debug-1-worker-0:217270:217270 [2] NCCL INFO 24 coll channels, 24 collnet channels, 0 nvls channels, 32 p2p channels, 32 p2p channels per peer +ywang29-p4d-debug-1-worker-0:217279:217279 [7] NCCL INFO TUNER/Plugin: Could not find: libnccl-tuner.so +ywang29-p4d-debug-1-worker-0:217279:217279 [7] NCCL INFO TUNER/Plugin: Using nccl_ofi_tuner (v3) +ywang29-p4d-debug-1-worker-0:217279:217279 [7] NCCL INFO NET/OFI NCCL_OFI_TUNER is not available for platform : p4de.24xlarge, Fall back to NCCL's tuner +ywang29-p4d-debug-1-worker-0:217279:217279 [7] NCCL INFO threadThresholds 8/8/64 | 64/8/64 | 512 | 512 +ywang29-p4d-debug-1-worker-0:217279:217279 [7] NCCL INFO 24 coll channels, 24 collnet channels, 0 nvls channels, 32 p2p channels, 32 p2p channels per peer +ywang29-p4d-debug-1-worker-0:217272:217272 [3] NCCL INFO TUNER/Plugin: Could not find: libnccl-tuner.so +ywang29-p4d-debug-1-worker-0:217272:217272 [3] NCCL INFO TUNER/Plugin: Using nccl_ofi_tuner (v3) +ywang29-p4d-debug-1-worker-0:217272:217272 [3] NCCL INFO NET/OFI NCCL_OFI_TUNER is not available for platform : p4de.24xlarge, Fall back to NCCL's tuner +ywang29-p4d-debug-1-worker-0:217272:217272 [3] NCCL INFO threadThresholds 8/8/64 | 64/8/64 | 512 | 512 +ywang29-p4d-debug-1-worker-0:217272:217272 [3] NCCL INFO 24 coll channels, 24 collnet channels, 0 nvls channels, 32 p2p channels, 32 p2p channels per peer +ywang29-p4d-debug-1-worker-0:217269:217269 [1] NCCL INFO TUNER/Plugin: Could not find: libnccl-tuner.so +ywang29-p4d-debug-1-worker-0:217269:217269 [1] NCCL INFO TUNER/Plugin: Using nccl_ofi_tuner (v3) +ywang29-p4d-debug-1-worker-0:217269:217269 [1] NCCL INFO NET/OFI NCCL_OFI_TUNER is not available for platform : p4de.24xlarge, Fall back to NCCL's tuner +ywang29-p4d-debug-1-worker-0:217269:217269 [1] NCCL INFO threadThresholds 8/8/64 | 64/8/64 | 512 | 512 +ywang29-p4d-debug-1-worker-0:217269:217269 [1] NCCL INFO 24 coll channels, 24 collnet channels, 0 nvls channels, 32 p2p channels, 32 p2p channels per peer +ywang29-p4d-debug-1-worker-0:217279:217279 [7] NCCL INFO ncclCommInitRankConfig comm 0x5556d7965790 rank 7 nranks 8 cudaDev 7 nvmlDev 7 busId a01d0 commId 0xf134627d563edc7d - Init COMPLETE +ywang29-p4d-debug-1-worker-0:217272:217272 [3] NCCL INFO ncclCommInitRankConfig comm 0x557edc8b65e0 rank 3 nranks 8 cudaDev 3 nvmlDev 3 busId 201d0 commId 0xf134627d563edc7d - Init COMPLETE +ywang29-p4d-debug-1-worker-0:217268:217268 [0] NCCL INFO ncclCommInitRankConfig comm 0x562c18cf2a40 rank 0 nranks 8 cudaDev 0 nvmlDev 0 busId 101c0 commId 0xf134627d563edc7d - Init COMPLETE +ywang29-p4d-debug-1-worker-0:217278:217278 [6] NCCL INFO ncclCommInitRankConfig comm 0x56514433a870 rank 6 nranks 8 cudaDev 6 nvmlDev 6 busId a01c0 commId 0xf134627d563edc7d - Init COMPLETE +ywang29-p4d-debug-1-worker-0:217275:217275 [5] NCCL INFO ncclCommInitRankConfig comm 0x55c53c3343a0 rank 5 nranks 8 cudaDev 5 nvmlDev 5 busId 901d0 commId 0xf134627d563edc7d - Init COMPLETE +ywang29-p4d-debug-1-worker-0:217270:217270 [2] NCCL INFO ncclCommInitRankConfig comm 0x55feadedf680 rank 2 nranks 8 cudaDev 2 nvmlDev 2 busId 201c0 commId 0xf134627d563edc7d - Init COMPLETE +ywang29-p4d-debug-1-worker-0:217274:217274 [4] NCCL INFO ncclCommInitRankConfig comm 0x56199da84a70 rank 4 nranks 8 cudaDev 4 nvmlDev 4 busId 901c0 commId 0xf134627d563edc7d - Init COMPLETE +ywang29-p4d-debug-1-worker-0:217272:217272 [3] NCCL INFO Init timings - ncclCommInitRankConfig: rank 3 nranks 8 total 0.40 (kernels 0.19, alloc 0.08, bootstrap 0.02, allgathers 0.00, topo 0.04, graphs 0.00, connections 0.06, rest 0.02) +ywang29-p4d-debug-1-worker-0:217268:217268 [0] NCCL INFO Init timings - ncclCommInitRankConfig: rank 0 nranks 8 total 0.46 (kernels 0.18, alloc 0.02, bootstrap 0.14, allgathers 0.00, topo 0.04, graphs 0.00, connections 0.05, rest 0.02) +ywang29-p4d-debug-1-worker-0:217279:217279 [7] NCCL INFO Init timings - ncclCommInitRankConfig: rank 7 nranks 8 total 0.44 (kernels 0.23, alloc 0.08, bootstrap 0.02, allgathers 0.00, topo 0.04, graphs 0.00, connections 0.06, rest 0.02) +ywang29-p4d-debug-1-worker-0:217275:217275 [5] NCCL INFO Init timings - ncclCommInitRankConfig: rank 5 nranks 8 total 0.40 (kernels 0.19, alloc 0.08, bootstrap 0.02, allgathers 0.01, topo 0.04, graphs 0.00, connections 0.05, rest 0.03) +ywang29-p4d-debug-1-worker-0:217269:217269 [1] NCCL INFO ncclCommInitRankConfig comm 0x55ef466ff050 rank 1 nranks 8 cudaDev 1 nvmlDev 1 busId 101d0 commId 0xf134627d563edc7d - Init COMPLETE +ywang29-p4d-debug-1-worker-0:217278:217278 [6] NCCL INFO Init timings - ncclCommInitRankConfig: rank 6 nranks 8 total 0.44 (kernels 0.18, alloc 0.04, bootstrap 0.10, allgathers 0.00, topo 0.04, graphs 0.00, connections 0.06, rest 0.02) +ywang29-p4d-debug-1-worker-0:217270:217270 [2] NCCL INFO Init timings - ncclCommInitRankConfig: rank 2 nranks 8 total 0.40 (kernels 0.19, alloc 0.08, bootstrap 0.01, allgathers 0.00, topo 0.04, graphs 0.00, connections 0.06, rest 0.02) +ywang29-p4d-debug-1-worker-0:217274:217274 [4] NCCL INFO Init timings - ncclCommInitRankConfig: rank 4 nranks 8 total 0.43 (kernels 0.18, alloc 0.04, bootstrap 0.09, allgathers 0.00, topo 0.04, graphs 0.00, connections 0.05, rest 0.03) +ywang29-p4d-debug-1-worker-0:217269:217269 [1] NCCL INFO Init timings - ncclCommInitRankConfig: rank 1 nranks 8 total 0.39 (kernels 0.25, alloc 0.02, bootstrap 0.00, allgathers 0.00, topo 0.04, graphs 0.00, connections 0.06, rest 0.02) +[INFO|2026-04-29 15:40:21] llamafactory.hparams.parser:505 >> Process rank: 4, world size: 8, device: cuda:4, distributed training: True, compute dtype: torch.bfloat16 +[INFO|2026-04-29 15:40:21] llamafactory.hparams.parser:505 >> Process rank: 6, world size: 8, device: cuda:6, distributed training: True, compute dtype: torch.bfloat16 +[INFO|2026-04-29 15:40:21] llamafactory.hparams.parser:505 >> Process rank: 2, world size: 8, device: cuda:2, distributed training: True, compute dtype: torch.bfloat16 +[INFO|2026-04-29 15:40:21] llamafactory.hparams.parser:505 >> Process rank: 0, world size: 8, device: cuda:0, distributed training: True, compute dtype: torch.bfloat16 +[INFO|2026-04-29 15:40:21] llamafactory.hparams.parser:505 >> Process rank: 7, world size: 8, device: cuda:7, distributed training: True, compute dtype: torch.bfloat16 +[INFO|2026-04-29 15:40:21] llamafactory.hparams.parser:505 >> Process rank: 3, world size: 8, device: cuda:3, distributed training: True, compute dtype: torch.bfloat16 +[INFO|2026-04-29 15:40:21] llamafactory.hparams.parser:505 >> Process rank: 5, world size: 8, device: cuda:5, distributed training: True, compute dtype: torch.bfloat16 +[INFO|2026-04-29 15:40:21] llamafactory.hparams.parser:505 >> Process rank: 1, world size: 8, device: cuda:1, distributed training: True, compute dtype: torch.bfloat16 +[INFO|configuration_utils.py:670] 2026-04-29 15:40:21,634 >> loading configuration file config.json from cache at /root/.cache/huggingface/hub/models--Qwen--Qwen3-1.7B/snapshots/70d244cc86ccca08cf5af4e1e306ecf908b1ad5e/config.json +[INFO|configuration_utils.py:742] 2026-04-29 15:40:21,637 >> Model config Qwen3Config { + "architectures": [ + "Qwen3ForCausalLM" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "bos_token_id": 151643, + "dtype": "bfloat16", + "eos_token_id": 151645, + "head_dim": 128, + "hidden_act": "silu", + "hidden_size": 2048, + "initializer_range": 0.02, + "intermediate_size": 6144, + "layer_types": [ + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention" + ], + "masked_layers": null, + "max_position_embeddings": 40960, + "max_window_layers": 28, + "model_type": "qwen3", + "num_attention_heads": 16, + "num_hidden_layers": 28, + "num_key_value_heads": 8, + "pad_token_id": null, + "rms_norm_eps": 1e-06, + "rope_parameters": { + "rope_theta": 1000000, + "rope_type": "default" + }, + "sliding_window": null, + "sparsity_attn": null, + "sparsity_mlp": null, + "subnet_mode": null, + "subnet_type": null, + "threshold_attn": null, + "threshold_mlp": null, + "tie_word_embeddings": true, + "transformers_version": "5.2.0", + "use_cache": true, + "use_sliding_window": false, + "vocab_size": 151936 +} + +[INFO|configuration_utils.py:670] 2026-04-29 15:40:24,077 >> loading configuration file config.json from cache at /root/.cache/huggingface/hub/models--Qwen--Qwen3-1.7B/snapshots/70d244cc86ccca08cf5af4e1e306ecf908b1ad5e/config.json +[INFO|configuration_utils.py:742] 2026-04-29 15:40:24,078 >> Model config Qwen3Config { + "architectures": [ + "Qwen3ForCausalLM" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "bos_token_id": 151643, + "dtype": "bfloat16", + "eos_token_id": 151645, + "head_dim": 128, + "hidden_act": "silu", + "hidden_size": 2048, + "initializer_range": 0.02, + "intermediate_size": 6144, + "layer_types": [ + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention" + ], + "masked_layers": null, + "max_position_embeddings": 40960, + "max_window_layers": 28, + "model_type": "qwen3", + "num_attention_heads": 16, + "num_hidden_layers": 28, + "num_key_value_heads": 8, + "pad_token_id": null, + "rms_norm_eps": 1e-06, + "rope_parameters": { + "rope_theta": 1000000, + "rope_type": "default" + }, + "sliding_window": null, + "sparsity_attn": null, + "sparsity_mlp": null, + "subnet_mode": null, + "subnet_type": null, + "threshold_attn": null, + "threshold_mlp": null, + "tie_word_embeddings": true, + "transformers_version": "5.2.0", + "use_cache": true, + "use_sliding_window": false, + "vocab_size": 151936 +} + +[INFO|configuration_utils.py:670] 2026-04-29 15:40:24,190 >> loading configuration file config.json from cache at /root/.cache/huggingface/hub/models--Qwen--Qwen3-1.7B/snapshots/70d244cc86ccca08cf5af4e1e306ecf908b1ad5e/config.json +[INFO|configuration_utils.py:742] 2026-04-29 15:40:24,191 >> Model config Qwen3Config { + "architectures": [ + "Qwen3ForCausalLM" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "bos_token_id": 151643, + "dtype": "bfloat16", + "eos_token_id": 151645, + "head_dim": 128, + "hidden_act": "silu", + "hidden_size": 2048, + "initializer_range": 0.02, + "intermediate_size": 6144, + "layer_types": [ + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention" + ], + "masked_layers": null, + "max_position_embeddings": 40960, + "max_window_layers": 28, + "model_type": "qwen3", + "num_attention_heads": 16, + "num_hidden_layers": 28, + "num_key_value_heads": 8, + "pad_token_id": null, + "rms_norm_eps": 1e-06, + "rope_parameters": { + "rope_theta": 1000000, + "rope_type": "default" + }, + "sliding_window": null, + "sparsity_attn": null, + "sparsity_mlp": null, + "subnet_mode": null, + "subnet_type": null, + "threshold_attn": null, + "threshold_mlp": null, + "tie_word_embeddings": true, + "transformers_version": "5.2.0", + "use_cache": true, + "use_sliding_window": false, + "vocab_size": 151936 +} + +ywang29-p4d-debug-1-worker-0:217270:217270 [2] NCCL INFO Channel 00/0 : 2[2] -> 3[3] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217270:217270 [2] NCCL INFO Channel 01/0 : 2[2] -> 3[3] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217270:217270 [2] NCCL INFO Channel 02/0 : 2[2] -> 3[3] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217270:217270 [2] NCCL INFO Channel 03/0 : 2[2] -> 3[3] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217270:217270 [2] NCCL INFO Channel 04/0 : 2[2] -> 3[3] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217270:217270 [2] NCCL INFO Channel 05/0 : 2[2] -> 3[3] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217270:217270 [2] NCCL INFO Channel 06/0 : 2[2] -> 3[3] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217270:217270 [2] NCCL INFO Channel 07/0 : 2[2] -> 3[3] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217270:217270 [2] NCCL INFO Channel 08/0 : 2[2] -> 3[3] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217270:217270 [2] NCCL INFO Channel 09/0 : 2[2] -> 3[3] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217270:217270 [2] NCCL INFO Channel 10/0 : 2[2] -> 3[3] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217270:217270 [2] NCCL INFO Channel 11/0 : 2[2] -> 3[3] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217270:217270 [2] NCCL INFO Channel 12/0 : 2[2] -> 3[3] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217270:217270 [2] NCCL INFO Channel 13/0 : 2[2] -> 3[3] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217270:217270 [2] NCCL INFO Channel 14/0 : 2[2] -> 3[3] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217270:217270 [2] NCCL INFO Channel 15/0 : 2[2] -> 3[3] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217270:217270 [2] NCCL INFO Channel 16/0 : 2[2] -> 3[3] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217270:217270 [2] NCCL INFO Channel 17/0 : 2[2] -> 3[3] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217270:217270 [2] NCCL INFO Channel 18/0 : 2[2] -> 3[3] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217270:217270 [2] NCCL INFO Channel 19/0 : 2[2] -> 3[3] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217270:217270 [2] NCCL INFO Channel 20/0 : 2[2] -> 3[3] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217270:217270 [2] NCCL INFO Channel 21/0 : 2[2] -> 3[3] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217270:217270 [2] NCCL INFO Channel 22/0 : 2[2] -> 3[3] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217270:217270 [2] NCCL INFO Channel 23/0 : 2[2] -> 3[3] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217272:217272 [3] NCCL INFO Channel 00/0 : 3[3] -> 4[4] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217272:217272 [3] NCCL INFO Channel 01/0 : 3[3] -> 4[4] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217272:217272 [3] NCCL INFO Channel 02/0 : 3[3] -> 4[4] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217272:217272 [3] NCCL INFO Channel 03/0 : 3[3] -> 4[4] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217272:217272 [3] NCCL INFO Channel 04/0 : 3[3] -> 4[4] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217272:217272 [3] NCCL INFO Channel 05/0 : 3[3] -> 4[4] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217272:217272 [3] NCCL INFO Channel 06/0 : 3[3] -> 4[4] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217272:217272 [3] NCCL INFO Channel 07/0 : 3[3] -> 4[4] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217272:217272 [3] NCCL INFO Channel 08/0 : 3[3] -> 4[4] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217272:217272 [3] NCCL INFO Channel 09/0 : 3[3] -> 4[4] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217272:217272 [3] NCCL INFO Channel 10/0 : 3[3] -> 4[4] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217272:217272 [3] NCCL INFO Channel 11/0 : 3[3] -> 4[4] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217272:217272 [3] NCCL INFO Channel 12/0 : 3[3] -> 4[4] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217272:217272 [3] NCCL INFO Channel 13/0 : 3[3] -> 4[4] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217272:217272 [3] NCCL INFO Channel 14/0 : 3[3] -> 4[4] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217272:217272 [3] NCCL INFO Channel 15/0 : 3[3] -> 4[4] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217272:217272 [3] NCCL INFO Channel 16/0 : 3[3] -> 4[4] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217272:217272 [3] NCCL INFO Channel 17/0 : 3[3] -> 4[4] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217272:217272 [3] NCCL INFO Channel 18/0 : 3[3] -> 4[4] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217272:217272 [3] NCCL INFO Channel 19/0 : 3[3] -> 4[4] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217272:217272 [3] NCCL INFO Channel 20/0 : 3[3] -> 4[4] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217272:217272 [3] NCCL INFO Channel 21/0 : 3[3] -> 4[4] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217272:217272 [3] NCCL INFO Channel 22/0 : 3[3] -> 4[4] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217272:217272 [3] NCCL INFO Channel 23/0 : 3[3] -> 4[4] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217275:217275 [5] NCCL INFO Channel 00/0 : 5[5] -> 6[6] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217275:217275 [5] NCCL INFO Channel 01/0 : 5[5] -> 6[6] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217275:217275 [5] NCCL INFO Channel 02/0 : 5[5] -> 6[6] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217275:217275 [5] NCCL INFO Channel 03/0 : 5[5] -> 6[6] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217275:217275 [5] NCCL INFO Channel 04/0 : 5[5] -> 6[6] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217275:217275 [5] NCCL INFO Channel 05/0 : 5[5] -> 6[6] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217275:217275 [5] NCCL INFO Channel 06/0 : 5[5] -> 6[6] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217275:217275 [5] NCCL INFO Channel 07/0 : 5[5] -> 6[6] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217275:217275 [5] NCCL INFO Channel 08/0 : 5[5] -> 6[6] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217275:217275 [5] NCCL INFO Channel 09/0 : 5[5] -> 6[6] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217275:217275 [5] NCCL INFO Channel 10/0 : 5[5] -> 6[6] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217275:217275 [5] NCCL INFO Channel 11/0 : 5[5] -> 6[6] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217275:217275 [5] NCCL INFO Channel 12/0 : 5[5] -> 6[6] via P2P/CUMEM/read +[INFO|2026-04-29 15:40:25] llamafactory.data.loader:144 >> Loading dataset allenai/tulu-3-sft-personas-code... +ywang29-p4d-debug-1-worker-0:217275:217275 [5] NCCL INFO Channel 13/0 : 5[5] -> 6[6] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217275:217275 [5] NCCL INFO Channel 14/0 : 5[5] -> 6[6] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217275:217275 [5] NCCL INFO Channel 15/0 : 5[5] -> 6[6] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217275:217275 [5] NCCL INFO Channel 16/0 : 5[5] -> 6[6] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217275:217275 [5] NCCL INFO Channel 17/0 : 5[5] -> 6[6] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217275:217275 [5] NCCL INFO Channel 18/0 : 5[5] -> 6[6] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217275:217275 [5] NCCL INFO Channel 19/0 : 5[5] -> 6[6] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217275:217275 [5] NCCL INFO Channel 20/0 : 5[5] -> 6[6] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217275:217275 [5] NCCL INFO Channel 21/0 : 5[5] -> 6[6] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217275:217275 [5] NCCL INFO Channel 22/0 : 5[5] -> 6[6] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217275:217275 [5] NCCL INFO Channel 23/0 : 5[5] -> 6[6] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217269:217269 [1] NCCL INFO Channel 00/0 : 1[1] -> 2[2] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217269:217269 [1] NCCL INFO Channel 01/0 : 1[1] -> 2[2] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217269:217269 [1] NCCL INFO Channel 02/0 : 1[1] -> 2[2] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217269:217269 [1] NCCL INFO Channel 03/0 : 1[1] -> 2[2] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217269:217269 [1] NCCL INFO Channel 04/0 : 1[1] -> 2[2] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217269:217269 [1] NCCL INFO Channel 05/0 : 1[1] -> 2[2] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217269:217269 [1] NCCL INFO Channel 06/0 : 1[1] -> 2[2] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217269:217269 [1] NCCL INFO Channel 07/0 : 1[1] -> 2[2] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217269:217269 [1] NCCL INFO Channel 08/0 : 1[1] -> 2[2] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217269:217269 [1] NCCL INFO Channel 09/0 : 1[1] -> 2[2] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217269:217269 [1] NCCL INFO Channel 10/0 : 1[1] -> 2[2] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217269:217269 [1] NCCL INFO Channel 11/0 : 1[1] -> 2[2] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217269:217269 [1] NCCL INFO Channel 12/0 : 1[1] -> 2[2] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217269:217269 [1] NCCL INFO Channel 13/0 : 1[1] -> 2[2] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217269:217269 [1] NCCL INFO Channel 14/0 : 1[1] -> 2[2] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217269:217269 [1] NCCL INFO Channel 15/0 : 1[1] -> 2[2] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217269:217269 [1] NCCL INFO Channel 16/0 : 1[1] -> 2[2] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217269:217269 [1] NCCL INFO Channel 17/0 : 1[1] -> 2[2] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217269:217269 [1] NCCL INFO Channel 18/0 : 1[1] -> 2[2] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217269:217269 [1] NCCL INFO Channel 19/0 : 1[1] -> 2[2] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217269:217269 [1] NCCL INFO Channel 20/0 : 1[1] -> 2[2] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217269:217269 [1] NCCL INFO Channel 21/0 : 1[1] -> 2[2] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217269:217269 [1] NCCL INFO Channel 22/0 : 1[1] -> 2[2] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217269:217269 [1] NCCL INFO Channel 23/0 : 1[1] -> 2[2] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217279:217279 [7] NCCL INFO Channel 00/0 : 7[7] -> 0[0] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217279:217279 [7] NCCL INFO Channel 01/0 : 7[7] -> 0[0] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217279:217279 [7] NCCL INFO Channel 02/0 : 7[7] -> 0[0] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217279:217279 [7] NCCL INFO Channel 03/0 : 7[7] -> 0[0] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217279:217279 [7] NCCL INFO Channel 04/0 : 7[7] -> 0[0] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217279:217279 [7] NCCL INFO Channel 05/0 : 7[7] -> 0[0] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217279:217279 [7] NCCL INFO Channel 06/0 : 7[7] -> 0[0] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217279:217279 [7] NCCL INFO Channel 07/0 : 7[7] -> 0[0] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217279:217279 [7] NCCL INFO Channel 08/0 : 7[7] -> 0[0] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217279:217279 [7] NCCL INFO Channel 09/0 : 7[7] -> 0[0] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217279:217279 [7] NCCL INFO Channel 10/0 : 7[7] -> 0[0] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217279:217279 [7] NCCL INFO Channel 11/0 : 7[7] -> 0[0] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217279:217279 [7] NCCL INFO Channel 12/0 : 7[7] -> 0[0] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217279:217279 [7] NCCL INFO Channel 13/0 : 7[7] -> 0[0] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217279:217279 [7] NCCL INFO Channel 14/0 : 7[7] -> 0[0] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217279:217279 [7] NCCL INFO Channel 15/0 : 7[7] -> 0[0] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217279:217279 [7] NCCL INFO Channel 16/0 : 7[7] -> 0[0] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217279:217279 [7] NCCL INFO Channel 17/0 : 7[7] -> 0[0] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217279:217279 [7] NCCL INFO Channel 18/0 : 7[7] -> 0[0] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217279:217279 [7] NCCL INFO Channel 19/0 : 7[7] -> 0[0] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217279:217279 [7] NCCL INFO Channel 20/0 : 7[7] -> 0[0] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217279:217279 [7] NCCL INFO Channel 21/0 : 7[7] -> 0[0] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217279:217279 [7] NCCL INFO Channel 22/0 : 7[7] -> 0[0] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217279:217279 [7] NCCL INFO Channel 23/0 : 7[7] -> 0[0] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217278:217278 [6] NCCL INFO Channel 00/0 : 6[6] -> 7[7] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217278:217278 [6] NCCL INFO Channel 01/0 : 6[6] -> 7[7] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217278:217278 [6] NCCL INFO Channel 02/0 : 6[6] -> 7[7] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217278:217278 [6] NCCL INFO Channel 03/0 : 6[6] -> 7[7] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217278:217278 [6] NCCL INFO Channel 04/0 : 6[6] -> 7[7] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217278:217278 [6] NCCL INFO Channel 05/0 : 6[6] -> 7[7] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217278:217278 [6] NCCL INFO Channel 06/0 : 6[6] -> 7[7] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217278:217278 [6] NCCL INFO Channel 07/0 : 6[6] -> 7[7] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217278:217278 [6] NCCL INFO Channel 08/0 : 6[6] -> 7[7] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217278:217278 [6] NCCL INFO Channel 09/0 : 6[6] -> 7[7] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217278:217278 [6] NCCL INFO Channel 10/0 : 6[6] -> 7[7] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217278:217278 [6] NCCL INFO Channel 11/0 : 6[6] -> 7[7] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217278:217278 [6] NCCL INFO Channel 12/0 : 6[6] -> 7[7] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217278:217278 [6] NCCL INFO Channel 13/0 : 6[6] -> 7[7] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217278:217278 [6] NCCL INFO Channel 14/0 : 6[6] -> 7[7] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217278:217278 [6] NCCL INFO Channel 15/0 : 6[6] -> 7[7] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217278:217278 [6] NCCL INFO Channel 16/0 : 6[6] -> 7[7] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217278:217278 [6] NCCL INFO Channel 17/0 : 6[6] -> 7[7] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217278:217278 [6] NCCL INFO Channel 18/0 : 6[6] -> 7[7] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217278:217278 [6] NCCL INFO Channel 19/0 : 6[6] -> 7[7] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217278:217278 [6] NCCL INFO Channel 20/0 : 6[6] -> 7[7] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217278:217278 [6] NCCL INFO Channel 21/0 : 6[6] -> 7[7] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217278:217278 [6] NCCL INFO Channel 22/0 : 6[6] -> 7[7] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217278:217278 [6] NCCL INFO Channel 23/0 : 6[6] -> 7[7] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217274:217274 [4] NCCL INFO Channel 00/0 : 4[4] -> 5[5] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217274:217274 [4] NCCL INFO Channel 01/0 : 4[4] -> 5[5] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217274:217274 [4] NCCL INFO Channel 02/0 : 4[4] -> 5[5] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217274:217274 [4] NCCL INFO Channel 03/0 : 4[4] -> 5[5] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217274:217274 [4] NCCL INFO Channel 04/0 : 4[4] -> 5[5] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217274:217274 [4] NCCL INFO Channel 05/0 : 4[4] -> 5[5] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217274:217274 [4] NCCL INFO Channel 06/0 : 4[4] -> 5[5] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217274:217274 [4] NCCL INFO Channel 07/0 : 4[4] -> 5[5] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217274:217274 [4] NCCL INFO Channel 08/0 : 4[4] -> 5[5] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217274:217274 [4] NCCL INFO Channel 09/0 : 4[4] -> 5[5] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217274:217274 [4] NCCL INFO Channel 10/0 : 4[4] -> 5[5] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217274:217274 [4] NCCL INFO Channel 11/0 : 4[4] -> 5[5] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217274:217274 [4] NCCL INFO Channel 12/0 : 4[4] -> 5[5] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217274:217274 [4] NCCL INFO Channel 13/0 : 4[4] -> 5[5] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217274:217274 [4] NCCL INFO Channel 14/0 : 4[4] -> 5[5] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217274:217274 [4] NCCL INFO Channel 15/0 : 4[4] -> 5[5] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217274:217274 [4] NCCL INFO Channel 16/0 : 4[4] -> 5[5] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217274:217274 [4] NCCL INFO Channel 17/0 : 4[4] -> 5[5] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217274:217274 [4] NCCL INFO Channel 18/0 : 4[4] -> 5[5] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217274:217274 [4] NCCL INFO Channel 19/0 : 4[4] -> 5[5] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217274:217274 [4] NCCL INFO Channel 20/0 : 4[4] -> 5[5] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217274:217274 [4] NCCL INFO Channel 21/0 : 4[4] -> 5[5] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217274:217274 [4] NCCL INFO Channel 22/0 : 4[4] -> 5[5] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217274:217274 [4] NCCL INFO Channel 23/0 : 4[4] -> 5[5] via P2P/CUMEM/read +[INFO|2026-04-29 15:40:26] llamafactory.data.loader:144 >> Sampled 10000 examples from dataset allenai/tulu-3-sft-personas-code. +[INFO|2026-04-29 15:40:26] llamafactory.data.loader:144 >> Loading dataset KKHYA/evol_codealpaca_converted... +Repo card metadata block was not found. Setting CardData to empty. +[INFO|2026-04-29 15:40:28] llamafactory.data.loader:144 >> Sampled 10000 examples from dataset KKHYA/evol_codealpaca_converted. +[INFO|2026-04-29 15:40:28] llamafactory.data.loader:144 >> Loading dataset KKHYA/codefeedback_filtered_instructions_converted... +Repo card metadata block was not found. Setting CardData to empty. +[INFO|2026-04-29 15:40:29] llamafactory.data.loader:144 >> Sampled 10000 examples from dataset KKHYA/codefeedback_filtered_instructions_converted. +ywang29-p4d-debug-1-worker-0:217268:217268 [0] NCCL INFO Channel 00/0 : 0[0] -> 1[1] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217268:217268 [0] NCCL INFO Channel 01/0 : 0[0] -> 1[1] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217268:217268 [0] NCCL INFO Channel 02/0 : 0[0] -> 1[1] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217268:217268 [0] NCCL INFO Channel 03/0 : 0[0] -> 1[1] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217268:217268 [0] NCCL INFO Channel 04/0 : 0[0] -> 1[1] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217268:217268 [0] NCCL INFO Channel 05/0 : 0[0] -> 1[1] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217268:217268 [0] NCCL INFO Channel 06/0 : 0[0] -> 1[1] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217268:217268 [0] NCCL INFO Channel 07/0 : 0[0] -> 1[1] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217268:217268 [0] NCCL INFO Channel 08/0 : 0[0] -> 1[1] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217268:217268 [0] NCCL INFO Channel 09/0 : 0[0] -> 1[1] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217268:217268 [0] NCCL INFO Channel 10/0 : 0[0] -> 1[1] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217268:217268 [0] NCCL INFO Channel 11/0 : 0[0] -> 1[1] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217268:217268 [0] NCCL INFO Channel 12/0 : 0[0] -> 1[1] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217268:217268 [0] NCCL INFO Channel 13/0 : 0[0] -> 1[1] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217268:217268 [0] NCCL INFO Channel 14/0 : 0[0] -> 1[1] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217268:217268 [0] NCCL INFO Channel 15/0 : 0[0] -> 1[1] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217268:217268 [0] NCCL INFO Channel 16/0 : 0[0] -> 1[1] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217268:217268 [0] NCCL INFO Channel 17/0 : 0[0] -> 1[1] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217268:217268 [0] NCCL INFO Channel 18/0 : 0[0] -> 1[1] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217268:217268 [0] NCCL INFO Channel 19/0 : 0[0] -> 1[1] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217268:217268 [0] NCCL INFO Channel 20/0 : 0[0] -> 1[1] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217268:217268 [0] NCCL INFO Channel 21/0 : 0[0] -> 1[1] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217268:217268 [0] NCCL INFO Channel 22/0 : 0[0] -> 1[1] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217268:217268 [0] NCCL INFO Channel 23/0 : 0[0] -> 1[1] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217269:217269 [1] NCCL INFO Connected all rings, use ring PXN 0 GDR 1 +ywang29-p4d-debug-1-worker-0:217275:217275 [5] NCCL INFO Connected all rings, use ring PXN 0 GDR 1 +ywang29-p4d-debug-1-worker-0:217268:217268 [0] NCCL INFO Connected all rings, use ring PXN 0 GDR 1 +ywang29-p4d-debug-1-worker-0:217279:217279 [7] NCCL INFO Connected all rings, use ring PXN 0 GDR 1 +ywang29-p4d-debug-1-worker-0:217274:217274 [4] NCCL INFO Connected all rings, use ring PXN 0 GDR 1 +ywang29-p4d-debug-1-worker-0:217278:217278 [6] NCCL INFO Connected all rings, use ring PXN 0 GDR 1 +ywang29-p4d-debug-1-worker-0:217270:217270 [2] NCCL INFO Connected all rings, use ring PXN 0 GDR 1 +ywang29-p4d-debug-1-worker-0:217272:217272 [3] NCCL INFO Connected all rings, use ring PXN 0 GDR 1 +training example: +input_ids: +[151644, 872, 198, 7985, 264, 10135, 729, 311, 11047, 279, 2790, 1372, 315, 8845, 16548, 553, 1674, 3767, 300, 4308, 304, 264, 2661, 3200, 504, 264, 1140, 315, 2432, 3059, 13, 8886, 2432, 1102, 374, 15251, 438, 264, 10997, 448, 6894, 330, 5117, 26532, 497, 330, 13757, 26532, 497, 330, 5117, 96244, 497, 323, 330, 13757, 96244, 3263, 1674, 3767, 300, 4308, 1410, 387, 2987, 279, 2114, 476, 3123, 2083, 304, 894, 2432, 13, 576, 1946, 374, 264, 1140, 315, 1741, 2432, 1102, 57514, 11, 323, 279, 2550, 1265, 387, 458, 7546, 14064, 279, 2790, 1372, 315, 8845, 16548, 553, 1674, 3767, 300, 4308, 382, 2505, 510, 12, 1565, 6347, 13576, 44622, 362, 1140, 315, 57514, 11, 1380, 1817, 10997, 5610, 510, 220, 481, 330, 5117, 26532, 1, 320, 917, 1648, 576, 829, 315, 279, 2114, 2083, 624, 220, 481, 330, 13757, 26532, 1, 320, 917, 1648, 576, 829, 315, 279, 3123, 2083, 624, 220, 481, 330, 5117, 96244, 1, 320, 396, 1648, 576, 1372, 315, 8845, 16548, 553, 279, 2114, 2083, 624, 220, 481, 330, 13757, 96244, 1, 320, 396, 1648, 576, 1372, 315, 8845, 16548, 553, 279, 3123, 2083, 382, 5097, 510, 12, 1527, 7546, 14064, 279, 2790, 1372, 315, 8845, 16548, 553, 1674, 3767, 300, 4308, 382, 13314, 510, 73594, 12669, 198, 6347, 13576, 284, 2278, 262, 5212, 5117, 26532, 788, 330, 2101, 3767, 300, 4308, 497, 330, 13757, 26532, 788, 330, 14597, 32, 497, 330, 5117, 96244, 788, 220, 17, 11, 330, 13757, 96244, 788, 220, 16, 1583, 262, 5212, 5117, 26532, 788, 330, 14597, 33, 497, 330, 13757, 26532, 788, 330, 2101, 3767, 300, 4308, 497, 330, 5117, 96244, 788, 220, 18, 11, 330, 13757, 96244, 788, 220, 17, 1583, 262, 5212, 5117, 26532, 788, 330, 2101, 3767, 300, 4308, 497, 330, 13757, 26532, 788, 330, 14597, 34, 497, 330, 5117, 96244, 788, 220, 16, 11, 330, 13757, 96244, 788, 220, 16, 1583, 262, 5212, 5117, 26532, 788, 330, 14597, 35, 497, 330, 13757, 26532, 788, 330, 2101, 3767, 300, 4308, 497, 330, 5117, 96244, 788, 220, 15, 11, 330, 13757, 96244, 788, 220, 18, 532, 921, 2, 31021, 9258, 25, 220, 23, 198, 73594, 151645, 198, 151644, 77091, 198, 750, 11047, 8418, 3767, 300, 4308, 96244, 25401, 13576, 982, 262, 2790, 96244, 284, 220, 15, 198, 262, 369, 2432, 304, 2432, 13576, 510, 286, 421, 2432, 1183, 5117, 26532, 1341, 621, 330, 2101, 3767, 300, 4308, 4660, 310, 2790, 96244, 1421, 2432, 1183, 5117, 96244, 7026, 286, 4409, 2432, 1183, 13757, 26532, 1341, 621, 330, 2101, 3767, 300, 4308, 4660, 310, 2790, 96244, 1421, 2432, 1183, 13757, 96244, 7026, 262, 470, 2790, 96244, 151645, 198] +inputs: +<|im_start|>user +Write a python function to calculate the total number of goals scored by Alanyaspor in a given season from a list of match results. Each match result is represented as a dictionary with keys "home_team", "away_team", "home_goals", and "away_goals". Alanyaspor could be either the home or away team in any match. The input is a list of such match result dictionaries, and the output should be an integer representing the total number of goals scored by Alanyaspor. + +Input: +- `match_results`: A list of dictionaries, where each dictionary contains: + - "home_team" (string): The name of the home team. + - "away_team" (string): The name of the away team. + - "home_goals" (int): The number of goals scored by the home team. + - "away_goals" (int): The number of goals scored by the away team. + +Output: +- An integer representing the total number of goals scored by Alanyaspor. + +Example: +```python +match_results = [ + {"home_team": "Alanyaspor", "away_team": "TeamA", "home_goals": 2, "away_goals": 1}, + {"home_team": "TeamB", "away_team": "Alanyaspor", "home_goals": 3, "away_goals": 2}, + {"home_team": "Alanyaspor", "away_team": "TeamC", "home_goals": 1, "away_goals": 1}, + {"home_team": "TeamD", "away_team": "Alanyaspor", "home_goals": 0, "away_goals": 3} +] +# Expected Output: 8 +```<|im_end|> +<|im_start|>assistant +def calculate_alanyaspor_goals(match_results): + total_goals = 0 + for match in match_results: + if match["home_team"] == "Alanyaspor": + total_goals += match["home_goals"] + elif match["away_team"] == "Alanyaspor": + total_goals += match["away_goals"] + return total_goals<|im_end|> + +label_ids: +[-100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, 750, 11047, 8418, 3767, 300, 4308, 96244, 25401, 13576, 982, 262, 2790, 96244, 284, 220, 15, 198, 262, 369, 2432, 304, 2432, 13576, 510, 286, 421, 2432, 1183, 5117, 26532, 1341, 621, 330, 2101, 3767, 300, 4308, 4660, 310, 2790, 96244, 1421, 2432, 1183, 5117, 96244, 7026, 286, 4409, 2432, 1183, 13757, 26532, 1341, 621, 330, 2101, 3767, 300, 4308, 4660, 310, 2790, 96244, 1421, 2432, 1183, 13757, 96244, 7026, 262, 470, 2790, 96244, 151645, 198] +labels: +def calculate_alanyaspor_goals(match_results): + total_goals = 0 + for match in match_results: + if match["home_team"] == "Alanyaspor": + total_goals += match["home_goals"] + elif match["away_team"] == "Alanyaspor": + total_goals += match["away_goals"] + return total_goals<|im_end|> + +Repo card metadata block was not found. Setting CardData to empty. +Repo card metadata block was not found. Setting CardData to empty. +Repo card metadata block was not found. Setting CardData to empty. +Repo card metadata block was not found. Setting CardData to empty. +Repo card metadata block was not found. Setting CardData to empty. +Repo card metadata block was not found. Setting CardData to empty. +Repo card metadata block was not found. Setting CardData to empty. +Repo card metadata block was not found. Setting CardData to empty. +Repo card metadata block was not found. Setting CardData to empty. +Repo card metadata block was not found. Setting CardData to empty. +Repo card metadata block was not found. Setting CardData to empty. +Repo card metadata block was not found. Setting CardData to empty. +Repo card metadata block was not found. Setting CardData to empty. +Repo card metadata block was not found. Setting CardData to empty. +[INFO|configuration_utils.py:670] 2026-04-29 15:40:33,156 >> loading configuration file config.json from cache at /root/.cache/huggingface/hub/models--Qwen--Qwen3-1.7B/snapshots/70d244cc86ccca08cf5af4e1e306ecf908b1ad5e/config.json +[INFO|configuration_utils.py:742] 2026-04-29 15:40:33,156 >> Model config Qwen3Config { + "architectures": [ + "Qwen3ForCausalLM" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "bos_token_id": 151643, + "dtype": "bfloat16", + "eos_token_id": 151645, + "head_dim": 128, + "hidden_act": "silu", + "hidden_size": 2048, + "initializer_range": 0.02, + "intermediate_size": 6144, + "layer_types": [ + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention" + ], + "masked_layers": null, + "max_position_embeddings": 40960, + "max_window_layers": 28, + "model_type": "qwen3", + "num_attention_heads": 16, + "num_hidden_layers": 28, + "num_key_value_heads": 8, + "pad_token_id": null, + "rms_norm_eps": 1e-06, + "rope_parameters": { + "rope_theta": 1000000, + "rope_type": "default" + }, + "sliding_window": null, + "sparsity_attn": null, + "sparsity_mlp": null, + "subnet_mode": null, + "subnet_type": null, + "threshold_attn": null, + "threshold_mlp": null, + "tie_word_embeddings": true, + "transformers_version": "5.2.0", + "use_cache": true, + "use_sliding_window": false, + "vocab_size": 151936 +} + +[INFO|2026-04-29 15:40:33] llamafactory.model.model_utils.kv_cache:144 >> KV cache is disabled during training. +[INFO|modeling_utils.py:710] 2026-04-29 15:40:33,644 >> loading weights file model.safetensors from cache at /root/.cache/huggingface/hub/models--Qwen--Qwen3-1.7B/snapshots/70d244cc86ccca08cf5af4e1e306ecf908b1ad5e/model.safetensors.index.json +[INFO|modeling_utils.py:779] 2026-04-29 15:40:33,644 >> Will use dtype=torch.bfloat16 as defined in model's config object +[INFO|configuration_utils.py:1014] 2026-04-29 15:40:33,645 >> Generate config GenerationConfig { + "bos_token_id": 151643, + "eos_token_id": 151645, + "output_attentions": false, + "output_hidden_states": false, + "use_cache": false +} + + Loading weights: 0%| | 0/311 [00:00> The tied weights mapping and config for this model specifies to tie model.embed_tokens.weight to lm_head.weight, but both are present in the checkpoints, so we will NOT tie them. You should update the config with `tie_word_embeddings=False` to silence this warning +[INFO|configuration_utils.py:967] 2026-04-29 15:40:34,721 >> loading configuration file generation_config.json from cache at /root/.cache/huggingface/hub/models--Qwen--Qwen3-1.7B/snapshots/70d244cc86ccca08cf5af4e1e306ecf908b1ad5e/generation_config.json +[INFO|configuration_utils.py:1014] 2026-04-29 15:40:34,721 >> Generate config GenerationConfig { + "bos_token_id": 151643, + "do_sample": true, + "eos_token_id": [ + 151645, + 151643 + ], + "pad_token_id": 151643, + "temperature": 0.6, + "top_k": 20, + "top_p": 0.95 +} + +[INFO|dynamic_module_utils.py:406] 2026-04-29 15:40:34,829 >> Could not locate the custom_generate/generate.py inside Qwen/Qwen3-1.7B. +[INFO|2026-04-29 15:40:34] llamafactory.model.model_utils.checkpointing:144 >> Gradient checkpointing enabled. +[INFO|2026-04-29 15:40:34] llamafactory.model.model_utils.attention:144 >> Using torch SDPA for faster training and inference. +[INFO|2026-04-29 15:40:34] llamafactory.model.adapter:144 >> Upcasting trainable params to float32. +[INFO|2026-04-29 15:40:34] llamafactory.model.adapter:144 >> Fine-tuning method: Full +[INFO|2026-04-29 15:40:34] llamafactory.model.loader:144 >> trainable params: 2,031,739,904 || all params: 2,031,739,904 || trainable%: 100.0000 +[WARNING|trainer_utils.py:1234] 2026-04-29 15:40:35,035 >> The tokenizer has new PAD/BOS/EOS tokens that differ from the model config and generation config. The model config and generation config were aligned accordingly, being updated with the tokenizer's values. Updated tokens: {'bos_token_id': None, 'pad_token_id': 151643}. + Loading weights: 0%| | 0/311 [00:002->1 [1] 3/-1/-1->2->1 [2] 3/-1/-1->2->1 [3] 3/-1/-1->2->1 [4] 3/-1/-1->2->1 [5] 3/-1/-1->2->1 [6] 3/-1/-1->2->1 [7] 3/-1/-1->2->1 [8] 3/-1/-1->2->1 [9] 3/-1/-1->2->1 [10] 3/-1/-1->2->1 [11] 3/-1/-1->2->1 [12] 3/-1/-1->2->1 [13] 3/-1/-1->2->1 [14] 3/-1/-1->2->1 [15] 3/-1/-1->2->1 [16] 3/-1/-1->2->1 [17] 3/-1/-1->2->1 [18] 3/-1/-1->2->1 [19] 3/-1/-1->2->1 [20] 3/-1/-1->2->1 [21] 3/-1/-1->2->1 [22] 3/-1/-1->2->1 [23] 3/-1/-1->2->1 +ywang29-p4d-debug-1-worker-0:217269:217269 [1] NCCL INFO Trees [0] 2/-1/-1->1->0 [1] 2/-1/-1->1->0 [2] 2/-1/-1->1->0 [3] 2/-1/-1->1->0 [4] 2/-1/-1->1->0 [5] 2/-1/-1->1->0 [6] 2/-1/-1->1->0 [7] 2/-1/-1->1->0 [8] 2/-1/-1->1->0 [9] 2/-1/-1->1->0 [10] 2/-1/-1->1->0 [11] 2/-1/-1->1->0 [12] 2/-1/-1->1->0 [13] 2/-1/-1->1->0 [14] 2/-1/-1->1->0 [15] 2/-1/-1->1->0 [16] 2/-1/-1->1->0 [17] 2/-1/-1->1->0 [18] 2/-1/-1->1->0 [19] 2/-1/-1->1->0 [20] 2/-1/-1->1->0 [21] 2/-1/-1->1->0 [22] 2/-1/-1->1->0 [23] 2/-1/-1->1->0 +ywang29-p4d-debug-1-worker-0:217268:217268 [0] NCCL INFO Channel 05/24 : 0 1 2 3 4 5 6 7 +ywang29-p4d-debug-1-worker-0:217270:217270 [2] NCCL INFO P2P Chunksize set to 524288 +ywang29-p4d-debug-1-worker-0:217269:217269 [1] NCCL INFO P2P Chunksize set to 524288 +ywang29-p4d-debug-1-worker-0:217268:217268 [0] NCCL INFO Channel 06/24 : 0 1 2 3 4 5 6 7 +ywang29-p4d-debug-1-worker-0:217268:217268 [0] NCCL INFO Channel 07/24 : 0 1 2 3 4 5 6 7 +ywang29-p4d-debug-1-worker-0:217268:217268 [0] NCCL INFO Channel 08/24 : 0 1 2 3 4 5 6 7 +ywang29-p4d-debug-1-worker-0:217268:217268 [0] NCCL INFO Channel 09/24 : 0 1 2 3 4 5 6 7 +ywang29-p4d-debug-1-worker-0:217272:217272 [3] NCCL INFO Trees [0] 4/-1/-1->3->2 [1] 4/-1/-1->3->2 [2] 4/-1/-1->3->2 [3] 4/-1/-1->3->2 [4] 4/-1/-1->3->2 [5] 4/-1/-1->3->2 [6] 4/-1/-1->3->2 [7] 4/-1/-1->3->2 [8] 4/-1/-1->3->2 [9] 4/-1/-1->3->2 [10] 4/-1/-1->3->2 [11] 4/-1/-1->3->2 [12] 4/-1/-1->3->2 [13] 4/-1/-1->3->2 [14] 4/-1/-1->3->2 [15] 4/-1/-1->3->2 [16] 4/-1/-1->3->2 [17] 4/-1/-1->3->2 [18] 4/-1/-1->3->2 [19] 4/-1/-1->3->2 [20] 4/-1/-1->3->2 [21] 4/-1/-1->3->2 [22] 4/-1/-1->3->2 [23] 4/-1/-1->3->2 +ywang29-p4d-debug-1-worker-0:217278:217278 [6] NCCL INFO Trees [0] 7/-1/-1->6->5 [1] 7/-1/-1->6->5 [2] 7/-1/-1->6->5 [3] 7/-1/-1->6->5 [4] 7/-1/-1->6->5 [5] 7/-1/-1->6->5 [6] 7/-1/-1->6->5 [7] 7/-1/-1->6->5 [8] 7/-1/-1->6->5 [9] 7/-1/-1->6->5 [10] 7/-1/-1->6->5 [11] 7/-1/-1->6->5 [12] 7/-1/-1->6->5 [13] 7/-1/-1->6->5 [14] 7/-1/-1->6->5 [15] 7/-1/-1->6->5 [16] 7/-1/-1->6->5 [17] 7/-1/-1->6->5 [18] 7/-1/-1->6->5 [19] 7/-1/-1->6->5 [20] 7/-1/-1->6->5 [21] 7/-1/-1->6->5 [22] 7/-1/-1->6->5 [23] 7/-1/-1->6->5 +ywang29-p4d-debug-1-worker-0:217279:217279 [7] NCCL INFO Trees [0] -1/-1/-1->7->6 [1] -1/-1/-1->7->6 [2] -1/-1/-1->7->6 [3] -1/-1/-1->7->6 [4] -1/-1/-1->7->6 [5] -1/-1/-1->7->6 [6] -1/-1/-1->7->6 [7] -1/-1/-1->7->6 [8] -1/-1/-1->7->6 [9] -1/-1/-1->7->6 [10] -1/-1/-1->7->6 [11] -1/-1/-1->7->6 [12] -1/-1/-1->7->6 [13] -1/-1/-1->7->6 [14] -1/-1/-1->7->6 [15] -1/-1/-1->7->6 [16] -1/-1/-1->7->6 [17] -1/-1/-1->7->6 [18] -1/-1/-1->7->6 [19] -1/-1/-1->7->6 [20] -1/-1/-1->7->6 [21] -1/-1/-1->7->6 [22] -1/-1/-1->7->6 [23] -1/-1/-1->7->6 +ywang29-p4d-debug-1-worker-0:217272:217272 [3] NCCL INFO P2P Chunksize set to 524288 +ywang29-p4d-debug-1-worker-0:217268:217268 [0] NCCL INFO Channel 10/24 : 0 1 2 3 4 5 6 7 +ywang29-p4d-debug-1-worker-0:217278:217278 [6] NCCL INFO P2P Chunksize set to 524288 +ywang29-p4d-debug-1-worker-0:217279:217279 [7] NCCL INFO P2P Chunksize set to 524288 +ywang29-p4d-debug-1-worker-0:217275:217275 [5] NCCL INFO Trees [0] 6/-1/-1->5->4 [1] 6/-1/-1->5->4 [2] 6/-1/-1->5->4 [3] 6/-1/-1->5->4 [4] 6/-1/-1->5->4 [5] 6/-1/-1->5->4 [6] 6/-1/-1->5->4 [7] 6/-1/-1->5->4 [8] 6/-1/-1->5->4 [9] 6/-1/-1->5->4 [10] 6/-1/-1->5->4 [11] 6/-1/-1->5->4 [12] 6/-1/-1->5->4 [13] 6/-1/-1->5->4 [14] 6/-1/-1->5->4 [15] 6/-1/-1->5->4 [16] 6/-1/-1->5->4 [17] 6/-1/-1->5->4 [18] 6/-1/-1->5->4 [19] 6/-1/-1->5->4 [20] 6/-1/-1->5->4 [21] 6/-1/-1->5->4 [22] 6/-1/-1->5->4 [23] 6/-1/-1->5->4 +ywang29-p4d-debug-1-worker-0:217268:217268 [0] NCCL INFO Channel 11/24 : 0 1 2 3 4 5 6 7 +ywang29-p4d-debug-1-worker-0:217275:217275 [5] NCCL INFO P2P Chunksize set to 524288 +ywang29-p4d-debug-1-worker-0:217268:217268 [0] NCCL INFO Channel 12/24 : 0 1 2 3 4 5 6 7 +ywang29-p4d-debug-1-worker-0:217268:217268 [0] NCCL INFO Channel 13/24 : 0 1 2 3 4 5 6 7 +ywang29-p4d-debug-1-worker-0:217268:217268 [0] NCCL INFO Channel 14/24 : 0 1 2 3 4 5 6 7 +ywang29-p4d-debug-1-worker-0:217268:217268 [0] NCCL INFO Channel 15/24 : 0 1 2 3 4 5 6 7 +ywang29-p4d-debug-1-worker-0:217268:217268 [0] NCCL INFO Channel 16/24 : 0 1 2 3 4 5 6 7 +ywang29-p4d-debug-1-worker-0:217268:217268 [0] NCCL INFO Channel 17/24 : 0 1 2 3 4 5 6 7 +ywang29-p4d-debug-1-worker-0:217268:217268 [0] NCCL INFO Channel 18/24 : 0 1 2 3 4 5 6 7 +ywang29-p4d-debug-1-worker-0:217268:217268 [0] NCCL INFO Channel 19/24 : 0 1 2 3 4 5 6 7 +ywang29-p4d-debug-1-worker-0:217268:217268 [0] NCCL INFO Channel 20/24 : 0 1 2 3 4 5 6 7 +ywang29-p4d-debug-1-worker-0:217268:217268 [0] NCCL INFO Channel 21/24 : 0 1 2 3 4 5 6 7 +ywang29-p4d-debug-1-worker-0:217268:217268 [0] NCCL INFO Channel 22/24 : 0 1 2 3 4 5 6 7 +ywang29-p4d-debug-1-worker-0:217274:217274 [4] NCCL INFO Trees [0] 5/-1/-1->4->3 [1] 5/-1/-1->4->3 [2] 5/-1/-1->4->3 [3] 5/-1/-1->4->3 [4] 5/-1/-1->4->3 [5] 5/-1/-1->4->3 [6] 5/-1/-1->4->3 [7] 5/-1/-1->4->3 [8] 5/-1/-1->4->3 [9] 5/-1/-1->4->3 [10] 5/-1/-1->4->3 [11] 5/-1/-1->4->3 [12] 5/-1/-1->4->3 [13] 5/-1/-1->4->3 [14] 5/-1/-1->4->3 [15] 5/-1/-1->4->3 [16] 5/-1/-1->4->3 [17] 5/-1/-1->4->3 [18] 5/-1/-1->4->3 [19] 5/-1/-1->4->3 [20] 5/-1/-1->4->3 [21] 5/-1/-1->4->3 [22] 5/-1/-1->4->3 [23] 5/-1/-1->4->3 +ywang29-p4d-debug-1-worker-0:217268:217268 [0] NCCL INFO Channel 23/24 : 0 1 2 3 4 5 6 7 +ywang29-p4d-debug-1-worker-0:217274:217274 [4] NCCL INFO P2P Chunksize set to 524288 +ywang29-p4d-debug-1-worker-0:217268:217268 [0] NCCL INFO Trees [0] 1/-1/-1->0->-1 [1] 1/-1/-1->0->-1 [2] 1/-1/-1->0->-1 [3] 1/-1/-1->0->-1 [4] 1/-1/-1->0->-1 [5] 1/-1/-1->0->-1 [6] 1/-1/-1->0->-1 [7] 1/-1/-1->0->-1 [8] 1/-1/-1->0->-1 [9] 1/-1/-1->0->-1 [10] 1/-1/-1->0->-1 [11] 1/-1/-1->0->-1 [12] 1/-1/-1->0->-1 [13] 1/-1/-1->0->-1 [14] 1/-1/-1->0->-1 [15] 1/-1/-1->0->-1 [16] 1/-1/-1->0->-1 [17] 1/-1/-1->0->-1 [18] 1/-1/-1->0->-1 [19] 1/-1/-1->0->-1 [20] 1/-1/-1->0->-1 [21] 1/-1/-1->0->-1 [22] 1/-1/-1->0->-1 [23] 1/-1/-1->0->-1 +ywang29-p4d-debug-1-worker-0:217268:217268 [0] NCCL INFO P2P Chunksize set to 524288 +ywang29-p4d-debug-1-worker-0:217279:217279 [7] NCCL INFO Check P2P Type isAllDirectP2p 1 directMode 0 isAllCudaP2p 1 +ywang29-p4d-debug-1-worker-0:217275:217275 [5] NCCL INFO Check P2P Type isAllDirectP2p 1 directMode 0 isAllCudaP2p 1 +ywang29-p4d-debug-1-worker-0:217278:217278 [6] NCCL INFO Check P2P Type isAllDirectP2p 1 directMode 0 isAllCudaP2p 1 +ywang29-p4d-debug-1-worker-0:217279:218055 [0] NCCL INFO [Proxy Service] Device 7 CPU core 33 +ywang29-p4d-debug-1-worker-0:217279:218056 [0] NCCL INFO [Proxy Service UDS] Device 7 CPU core 35 +ywang29-p4d-debug-1-worker-0:217275:218057 [0] NCCL INFO [Proxy Service] Device 5 CPU core 34 +ywang29-p4d-debug-1-worker-0:217275:218058 [0] NCCL INFO [Proxy Service UDS] Device 5 CPU core 38 +ywang29-p4d-debug-1-worker-0:217278:218059 [0] NCCL INFO [Proxy Service] Device 6 CPU core 38 +ywang29-p4d-debug-1-worker-0:217278:218060 [0] NCCL INFO [Proxy Service UDS] Device 6 CPU core 93 +ywang29-p4d-debug-1-worker-0:217269:217269 [1] NCCL INFO Check P2P Type isAllDirectP2p 1 directMode 0 isAllCudaP2p 1 +ywang29-p4d-debug-1-worker-0:217269:218061 [0] NCCL INFO [Proxy Service] Device 1 CPU core 22 +ywang29-p4d-debug-1-worker-0:217269:218062 [0] NCCL INFO [Proxy Service UDS] Device 1 CPU core 23 +ywang29-p4d-debug-1-worker-0:217272:217272 [3] NCCL INFO Check P2P Type isAllDirectP2p 1 directMode 0 isAllCudaP2p 1 +ywang29-p4d-debug-1-worker-0:217272:218063 [0] NCCL INFO [Proxy Service] Device 3 CPU core 15 +ywang29-p4d-debug-1-worker-0:217272:218064 [0] NCCL INFO [Proxy Service UDS] Device 3 CPU core 65 +ywang29-p4d-debug-1-worker-0:217268:217268 [0] NCCL INFO Check P2P Type isAllDirectP2p 1 directMode 0 isAllCudaP2p 1 +ywang29-p4d-debug-1-worker-0:217268:218065 [0] NCCL INFO [Proxy Service] Device 0 CPU core 66 +ywang29-p4d-debug-1-worker-0:217268:218066 [0] NCCL INFO [Proxy Service UDS] Device 0 CPU core 19 +ywang29-p4d-debug-1-worker-0:217274:217274 [4] NCCL INFO Check P2P Type isAllDirectP2p 1 directMode 0 isAllCudaP2p 1 +ywang29-p4d-debug-1-worker-0:217274:218067 [0] NCCL INFO [Proxy Service] Device 4 CPU core 28 +ywang29-p4d-debug-1-worker-0:217274:218068 [0] NCCL INFO [Proxy Service UDS] Device 4 CPU core 32 +ywang29-p4d-debug-1-worker-0:217270:217270 [2] NCCL INFO Check P2P Type isAllDirectP2p 1 directMode 0 isAllCudaP2p 1 +ywang29-p4d-debug-1-worker-0:217270:218069 [0] NCCL INFO [Proxy Service] Device 2 CPU core 50 +ywang29-p4d-debug-1-worker-0:217270:218070 [0] NCCL INFO [Proxy Service UDS] Device 2 CPU core 55 +ywang29-p4d-debug-1-worker-0:217278:217278 [6] NCCL INFO NET/OFI NCCL_OFI_TUNER is not available for platform : p4de.24xlarge, Fall back to NCCL's tuner +ywang29-p4d-debug-1-worker-0:217278:217278 [6] NCCL INFO threadThresholds 8/8/64 | 64/8/64 | 512 | 512 +ywang29-p4d-debug-1-worker-0:217278:217278 [6] NCCL INFO 24 coll channels, 24 collnet channels, 0 nvls channels, 32 p2p channels, 32 p2p channels per peer +ywang29-p4d-debug-1-worker-0:217279:217279 [7] NCCL INFO NET/OFI NCCL_OFI_TUNER is not available for platform : p4de.24xlarge, Fall back to NCCL's tuner +ywang29-p4d-debug-1-worker-0:217279:217279 [7] NCCL INFO threadThresholds 8/8/64 | 64/8/64 | 512 | 512 +ywang29-p4d-debug-1-worker-0:217279:217279 [7] NCCL INFO 24 coll channels, 24 collnet channels, 0 nvls channels, 32 p2p channels, 32 p2p channels per peer +ywang29-p4d-debug-1-worker-0:217269:217269 [1] NCCL INFO NET/OFI NCCL_OFI_TUNER is not available for platform : p4de.24xlarge, Fall back to NCCL's tuner +ywang29-p4d-debug-1-worker-0:217269:217269 [1] NCCL INFO threadThresholds 8/8/64 | 64/8/64 | 512 | 512 +ywang29-p4d-debug-1-worker-0:217269:217269 [1] NCCL INFO 24 coll channels, 24 collnet channels, 0 nvls channels, 32 p2p channels, 32 p2p channels per peer +ywang29-p4d-debug-1-worker-0:217274:217274 [4] NCCL INFO NET/OFI NCCL_OFI_TUNER is not available for platform : p4de.24xlarge, Fall back to NCCL's tuner +ywang29-p4d-debug-1-worker-0:217274:217274 [4] NCCL INFO threadThresholds 8/8/64 | 64/8/64 | 512 | 512 +ywang29-p4d-debug-1-worker-0:217274:217274 [4] NCCL INFO 24 coll channels, 24 collnet channels, 0 nvls channels, 32 p2p channels, 32 p2p channels per peer +ywang29-p4d-debug-1-worker-0:217275:217275 [5] NCCL INFO NET/OFI NCCL_OFI_TUNER is not available for platform : p4de.24xlarge, Fall back to NCCL's tuner +ywang29-p4d-debug-1-worker-0:217275:217275 [5] NCCL INFO threadThresholds 8/8/64 | 64/8/64 | 512 | 512 +ywang29-p4d-debug-1-worker-0:217275:217275 [5] NCCL INFO 24 coll channels, 24 collnet channels, 0 nvls channels, 32 p2p channels, 32 p2p channels per peer +ywang29-p4d-debug-1-worker-0:217272:217272 [3] NCCL INFO NET/OFI NCCL_OFI_TUNER is not available for platform : p4de.24xlarge, Fall back to NCCL's tuner +ywang29-p4d-debug-1-worker-0:217272:217272 [3] NCCL INFO threadThresholds 8/8/64 | 64/8/64 | 512 | 512 +ywang29-p4d-debug-1-worker-0:217272:217272 [3] NCCL INFO 24 coll channels, 24 collnet channels, 0 nvls channels, 32 p2p channels, 32 p2p channels per peer +ywang29-p4d-debug-1-worker-0:217268:217268 [0] NCCL INFO NET/OFI NCCL_OFI_TUNER is not available for platform : p4de.24xlarge, Fall back to NCCL's tuner +ywang29-p4d-debug-1-worker-0:217268:217268 [0] NCCL INFO threadThresholds 8/8/64 | 64/8/64 | 512 | 512 +ywang29-p4d-debug-1-worker-0:217268:217268 [0] NCCL INFO 24 coll channels, 24 collnet channels, 0 nvls channels, 32 p2p channels, 32 p2p channels per peer +ywang29-p4d-debug-1-worker-0:217268:217268 [0] NCCL INFO CC Off, workFifoBytes 1048576 +ywang29-p4d-debug-1-worker-0:217270:217270 [2] NCCL INFO NET/OFI NCCL_OFI_TUNER is not available for platform : p4de.24xlarge, Fall back to NCCL's tuner +ywang29-p4d-debug-1-worker-0:217270:217270 [2] NCCL INFO threadThresholds 8/8/64 | 64/8/64 | 512 | 512 +ywang29-p4d-debug-1-worker-0:217270:217270 [2] NCCL INFO 24 coll channels, 24 collnet channels, 0 nvls channels, 32 p2p channels, 32 p2p channels per peer +ywang29-p4d-debug-1-worker-0:217269:217269 [1] NCCL INFO ncclCommSplit comm 0x55ef4f96de00 rank 1 nranks 8 cudaDev 1 nvmlDev 1 busId 101d0 parent 0x55ef466ff050 splitCount 1 color 1266629538 key 1 - Init COMPLETE +ywang29-p4d-debug-1-worker-0:217275:217275 [5] NCCL INFO ncclCommSplit comm 0x55c5481be410 rank 5 nranks 8 cudaDev 5 nvmlDev 5 busId 901d0 parent 0x55c53c3343a0 splitCount 1 color 1266629538 key 5 - Init COMPLETE +ywang29-p4d-debug-1-worker-0:217279:217279 [7] NCCL INFO ncclCommSplit comm 0x5556e3718090 rank 7 nranks 8 cudaDev 7 nvmlDev 7 busId a01d0 parent 0x5556d7965790 splitCount 1 color 1266629538 key 7 - Init COMPLETE +ywang29-p4d-debug-1-worker-0:217272:217272 [3] NCCL INFO ncclCommSplit comm 0x557ee87f3a90 rank 3 nranks 8 cudaDev 3 nvmlDev 3 busId 201d0 parent 0x557edc8b65e0 splitCount 1 color 1266629538 key 3 - Init COMPLETE +ywang29-p4d-debug-1-worker-0:217269:217269 [1] NCCL INFO Init timings - ncclCommSplit: rank 1 nranks 8 total 0.70 (kernels 0.00, alloc 0.00, bootstrap 0.00, allgathers 0.01, topo 0.03, graphs 0.00, connections 0.06, rest 0.60) +ywang29-p4d-debug-1-worker-0:217275:217275 [5] NCCL INFO Init timings - ncclCommSplit: rank 5 nranks 8 total 0.68 (kernels 0.00, alloc 0.00, bootstrap 0.00, allgathers 0.01, topo 0.03, graphs 0.00, connections 0.07, rest 0.58) +ywang29-p4d-debug-1-worker-0:217279:217279 [7] NCCL INFO Init timings - ncclCommSplit: rank 7 nranks 8 total 0.13 (kernels 0.00, alloc 0.00, bootstrap 0.00, allgathers 0.00, topo 0.04, graphs 0.00, connections 0.06, rest 0.03) +ywang29-p4d-debug-1-worker-0:217272:217272 [3] NCCL INFO Init timings - ncclCommSplit: rank 3 nranks 8 total 0.65 (kernels 0.00, alloc 0.00, bootstrap 0.00, allgathers 0.01, topo 0.03, graphs 0.00, connections 0.06, rest 0.54) +ywang29-p4d-debug-1-worker-0:217278:217278 [6] NCCL INFO ncclCommSplit comm 0x56515020cac0 rank 6 nranks 8 cudaDev 6 nvmlDev 6 busId a01c0 parent 0x56514433a870 splitCount 1 color 1266629538 key 6 - Init COMPLETE +ywang29-p4d-debug-1-worker-0:217270:217270 [2] NCCL INFO ncclCommSplit comm 0x55febaf71640 rank 2 nranks 8 cudaDev 2 nvmlDev 2 busId 201c0 parent 0x55feadedf680 splitCount 1 color 1266629538 key 2 - Init COMPLETE +ywang29-p4d-debug-1-worker-0:217278:217278 [6] NCCL INFO Init timings - ncclCommSplit: rank 6 nranks 8 total 0.45 (kernels 0.00, alloc 0.00, bootstrap 0.00, allgathers 0.00, topo 0.04, graphs 0.00, connections 0.05, rest 0.35) +ywang29-p4d-debug-1-worker-0:217270:217270 [2] NCCL INFO Init timings - ncclCommSplit: rank 2 nranks 8 total 0.35 (kernels 0.00, alloc 0.00, bootstrap 0.00, allgathers 0.00, topo 0.04, graphs 0.00, connections 0.06, rest 0.25) +ywang29-p4d-debug-1-worker-0:217268:217268 [0] NCCL INFO ncclCommSplit comm 0x562c26858c80 rank 0 nranks 8 cudaDev 0 nvmlDev 0 busId 101c0 parent 0x562c18cf2a40 splitCount 1 color 1266629538 key 0 - Init COMPLETE +ywang29-p4d-debug-1-worker-0:217268:217268 [0] NCCL INFO Init timings - ncclCommSplit: rank 0 nranks 8 total 2.21 (kernels 0.00, alloc 0.00, bootstrap 0.00, allgathers 0.01, topo 0.03, graphs 0.00, connections 0.06, rest 2.10) +ywang29-p4d-debug-1-worker-0:217274:217274 [4] NCCL INFO ncclCommSplit comm 0x5619aa0bfad0 rank 4 nranks 8 cudaDev 4 nvmlDev 4 busId 901c0 parent 0x56199da84a70 splitCount 1 color 1266629538 key 4 - Init COMPLETE +ywang29-p4d-debug-1-worker-0:217274:217274 [4] NCCL INFO Init timings - ncclCommSplit: rank 4 nranks 8 total 0.69 (kernels 0.00, alloc 0.00, bootstrap 0.00, allgathers 0.01, topo 0.03, graphs 0.00, connections 0.06, rest 0.59) +ywang29-p4d-debug-1-worker-0:217274:217274 [4] NCCL INFO Channel 00/0 : 4[4] -> 5[5] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217279:217279 [7] NCCL INFO Channel 00/0 : 7[7] -> 0[0] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217278:217278 [6] NCCL INFO Channel 00/0 : 6[6] -> 7[7] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217278:217278 [6] NCCL INFO Channel 01/0 : 6[6] -> 7[7] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217272:217272 [3] NCCL INFO Channel 00/0 : 3[3] -> 4[4] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217274:217274 [4] NCCL INFO Channel 01/0 : 4[4] -> 5[5] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217279:217279 [7] NCCL INFO Channel 01/0 : 7[7] -> 0[0] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217274:217274 [4] NCCL INFO Channel 02/0 : 4[4] -> 5[5] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217269:217269 [1] NCCL INFO Channel 00/0 : 1[1] -> 2[2] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217279:217279 [7] NCCL INFO Channel 02/0 : 7[7] -> 0[0] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217272:217272 [3] NCCL INFO Channel 01/0 : 3[3] -> 4[4] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217278:217278 [6] NCCL INFO Channel 02/0 : 6[6] -> 7[7] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217272:217272 [3] NCCL INFO Channel 02/0 : 3[3] -> 4[4] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217269:217269 [1] NCCL INFO Channel 01/0 : 1[1] -> 2[2] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217278:217278 [6] NCCL INFO Channel 03/0 : 6[6] -> 7[7] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217268:217268 [0] NCCL INFO Channel 00/0 : 0[0] -> 1[1] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217279:217279 [7] NCCL INFO Channel 03/0 : 7[7] -> 0[0] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217272:217272 [3] NCCL INFO Channel 03/0 : 3[3] -> 4[4] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217274:217274 [4] NCCL INFO Channel 03/0 : 4[4] -> 5[5] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217279:217279 [7] NCCL INFO Channel 04/0 : 7[7] -> 0[0] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217269:217269 [1] NCCL INFO Channel 02/0 : 1[1] -> 2[2] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217278:217278 [6] NCCL INFO Channel 04/0 : 6[6] -> 7[7] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217268:217268 [0] NCCL INFO Channel 01/0 : 0[0] -> 1[1] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217275:217275 [5] NCCL INFO Channel 00/0 : 5[5] -> 6[6] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217268:217268 [0] NCCL INFO Channel 02/0 : 0[0] -> 1[1] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217278:217278 [6] NCCL INFO Channel 05/0 : 6[6] -> 7[7] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217279:217279 [7] NCCL INFO Channel 05/0 : 7[7] -> 0[0] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217269:217269 [1] NCCL INFO Channel 03/0 : 1[1] -> 2[2] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217274:217274 [4] NCCL INFO Channel 04/0 : 4[4] -> 5[5] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217274:217274 [4] NCCL INFO Channel 05/0 : 4[4] -> 5[5] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217272:217272 [3] NCCL INFO Channel 04/0 : 3[3] -> 4[4] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217279:217279 [7] NCCL INFO Channel 06/0 : 7[7] -> 0[0] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217270:217270 [2] NCCL INFO Channel 00/0 : 2[2] -> 3[3] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217278:217278 [6] NCCL INFO Channel 06/0 : 6[6] -> 7[7] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217269:217269 [1] NCCL INFO Channel 04/0 : 1[1] -> 2[2] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217275:217275 [5] NCCL INFO Channel 01/0 : 5[5] -> 6[6] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217270:217270 [2] NCCL INFO Channel 01/0 : 2[2] -> 3[3] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217274:217274 [4] NCCL INFO Channel 06/0 : 4[4] -> 5[5] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217269:217269 [1] NCCL INFO Channel 05/0 : 1[1] -> 2[2] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217279:217279 [7] NCCL INFO Channel 07/0 : 7[7] -> 0[0] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217278:217278 [6] NCCL INFO Channel 07/0 : 6[6] -> 7[7] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217268:217268 [0] NCCL INFO Channel 03/0 : 0[0] -> 1[1] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217272:217272 [3] NCCL INFO Channel 05/0 : 3[3] -> 4[4] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217274:217274 [4] NCCL INFO Channel 07/0 : 4[4] -> 5[5] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217269:217269 [1] NCCL INFO Channel 06/0 : 1[1] -> 2[2] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217279:217279 [7] NCCL INFO Channel 08/0 : 7[7] -> 0[0] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217278:217278 [6] NCCL INFO Channel 08/0 : 6[6] -> 7[7] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217278:217278 [6] NCCL INFO Channel 09/0 : 6[6] -> 7[7] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217270:217270 [2] NCCL INFO Channel 02/0 : 2[2] -> 3[3] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217269:217269 [1] NCCL INFO Channel 07/0 : 1[1] -> 2[2] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217274:217274 [4] NCCL INFO Channel 08/0 : 4[4] -> 5[5] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217279:217279 [7] NCCL INFO Channel 09/0 : 7[7] -> 0[0] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217275:217275 [5] NCCL INFO Channel 02/0 : 5[5] -> 6[6] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217275:217275 [5] NCCL INFO Channel 03/0 : 5[5] -> 6[6] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217274:217274 [4] NCCL INFO Channel 09/0 : 4[4] -> 5[5] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217272:217272 [3] NCCL INFO Channel 06/0 : 3[3] -> 4[4] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217279:217279 [7] NCCL INFO Channel 10/0 : 7[7] -> 0[0] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217268:217268 [0] NCCL INFO Channel 04/0 : 0[0] -> 1[1] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217270:217270 [2] NCCL INFO Channel 03/0 : 2[2] -> 3[3] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217269:217269 [1] NCCL INFO Channel 08/0 : 1[1] -> 2[2] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217269:217269 [1] NCCL INFO Channel 09/0 : 1[1] -> 2[2] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217272:217272 [3] NCCL INFO Channel 07/0 : 3[3] -> 4[4] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217268:217268 [0] NCCL INFO Channel 05/0 : 0[0] -> 1[1] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217279:217279 [7] NCCL INFO Channel 11/0 : 7[7] -> 0[0] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217270:217270 [2] NCCL INFO Channel 04/0 : 2[2] -> 3[3] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217274:217274 [4] NCCL INFO Channel 10/0 : 4[4] -> 5[5] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217274:217274 [4] NCCL INFO Channel 11/0 : 4[4] -> 5[5] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217268:217268 [0] NCCL INFO Channel 06/0 : 0[0] -> 1[1] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217278:217278 [6] NCCL INFO Channel 10/0 : 6[6] -> 7[7] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217270:217270 [2] NCCL INFO Channel 05/0 : 2[2] -> 3[3] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217279:217279 [7] NCCL INFO Channel 12/0 : 7[7] -> 0[0] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217275:217275 [5] NCCL INFO Channel 04/0 : 5[5] -> 6[6] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217279:217279 [7] NCCL INFO Channel 13/0 : 7[7] -> 0[0] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217269:217269 [1] NCCL INFO Channel 10/0 : 1[1] -> 2[2] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217272:217272 [3] NCCL INFO Channel 08/0 : 3[3] -> 4[4] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217278:217278 [6] NCCL INFO Channel 11/0 : 6[6] -> 7[7] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217270:217270 [2] NCCL INFO Channel 06/0 : 2[2] -> 3[3] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217275:217275 [5] NCCL INFO Channel 05/0 : 5[5] -> 6[6] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217274:217274 [4] NCCL INFO Channel 12/0 : 4[4] -> 5[5] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217279:217279 [7] NCCL INFO Channel 14/0 : 7[7] -> 0[0] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217268:217268 [0] NCCL INFO Channel 07/0 : 0[0] -> 1[1] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217278:217278 [6] NCCL INFO Channel 12/0 : 6[6] -> 7[7] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217270:217270 [2] NCCL INFO Channel 07/0 : 2[2] -> 3[3] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217274:217274 [4] NCCL INFO Channel 13/0 : 4[4] -> 5[5] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217275:217275 [5] NCCL INFO Channel 06/0 : 5[5] -> 6[6] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217269:217269 [1] NCCL INFO Channel 11/0 : 1[1] -> 2[2] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217275:217275 [5] NCCL INFO Channel 07/0 : 5[5] -> 6[6] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217272:217272 [3] NCCL INFO Channel 09/0 : 3[3] -> 4[4] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217274:217274 [4] NCCL INFO Channel 14/0 : 4[4] -> 5[5] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217270:217270 [2] NCCL INFO Channel 08/0 : 2[2] -> 3[3] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217278:217278 [6] NCCL INFO Channel 13/0 : 6[6] -> 7[7] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217278:217278 [6] NCCL INFO Channel 14/0 : 6[6] -> 7[7] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217279:217279 [7] NCCL INFO Channel 15/0 : 7[7] -> 0[0] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217272:217272 [3] NCCL INFO Channel 10/0 : 3[3] -> 4[4] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217268:217268 [0] NCCL INFO Channel 08/0 : 0[0] -> 1[1] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217274:217274 [4] NCCL INFO Channel 15/0 : 4[4] -> 5[5] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217270:217270 [2] NCCL INFO Channel 09/0 : 2[2] -> 3[3] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217279:217279 [7] NCCL INFO Channel 16/0 : 7[7] -> 0[0] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217272:217272 [3] NCCL INFO Channel 11/0 : 3[3] -> 4[4] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217274:217274 [4] NCCL INFO Channel 16/0 : 4[4] -> 5[5] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217270:217270 [2] NCCL INFO Channel 10/0 : 2[2] -> 3[3] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217269:217269 [1] NCCL INFO Channel 12/0 : 1[1] -> 2[2] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217268:217268 [0] NCCL INFO Channel 09/0 : 0[0] -> 1[1] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217268:217268 [0] NCCL INFO Channel 10/0 : 0[0] -> 1[1] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217269:217269 [1] NCCL INFO Channel 13/0 : 1[1] -> 2[2] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217275:217275 [5] NCCL INFO Channel 08/0 : 5[5] -> 6[6] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217270:217270 [2] NCCL INFO Channel 11/0 : 2[2] -> 3[3] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217278:217278 [6] NCCL INFO Channel 15/0 : 6[6] -> 7[7] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217274:217274 [4] NCCL INFO Channel 17/0 : 4[4] -> 5[5] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217274:217274 [4] NCCL INFO Channel 18/0 : 4[4] -> 5[5] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217270:217270 [2] NCCL INFO Channel 12/0 : 2[2] -> 3[3] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217272:217272 [3] NCCL INFO Channel 12/0 : 3[3] -> 4[4] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217279:217279 [7] NCCL INFO Channel 17/0 : 7[7] -> 0[0] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217278:217278 [6] NCCL INFO Channel 16/0 : 6[6] -> 7[7] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217275:217275 [5] NCCL INFO Channel 09/0 : 5[5] -> 6[6] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217275:217275 [5] NCCL INFO Channel 10/0 : 5[5] -> 6[6] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217278:217278 [6] NCCL INFO Channel 17/0 : 6[6] -> 7[7] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217268:217268 [0] NCCL INFO Channel 11/0 : 0[0] -> 1[1] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217279:217279 [7] NCCL INFO Channel 18/0 : 7[7] -> 0[0] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217272:217272 [3] NCCL INFO Channel 13/0 : 3[3] -> 4[4] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217269:217269 [1] NCCL INFO Channel 14/0 : 1[1] -> 2[2] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217270:217270 [2] NCCL INFO Channel 13/0 : 2[2] -> 3[3] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217270:217270 [2] NCCL INFO Channel 14/0 : 2[2] -> 3[3] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217272:217272 [3] NCCL INFO Channel 14/0 : 3[3] -> 4[4] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217274:217274 [4] NCCL INFO Channel 19/0 : 4[4] -> 5[5] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217269:217269 [1] NCCL INFO Channel 15/0 : 1[1] -> 2[2] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217279:217279 [7] NCCL INFO Channel 19/0 : 7[7] -> 0[0] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217275:217275 [5] NCCL INFO Channel 11/0 : 5[5] -> 6[6] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217275:217275 [5] NCCL INFO Channel 12/0 : 5[5] -> 6[6] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217269:217269 [1] NCCL INFO Channel 16/0 : 1[1] -> 2[2] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217278:217278 [6] NCCL INFO Channel 18/0 : 6[6] -> 7[7] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217268:217268 [0] NCCL INFO Channel 12/0 : 0[0] -> 1[1] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217274:217274 [4] NCCL INFO Channel 20/0 : 4[4] -> 5[5] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217279:217279 [7] NCCL INFO Channel 20/0 : 7[7] -> 0[0] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217270:217270 [2] NCCL INFO Channel 15/0 : 2[2] -> 3[3] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217272:217272 [3] NCCL INFO Channel 15/0 : 3[3] -> 4[4] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217279:217279 [7] NCCL INFO Channel 21/0 : 7[7] -> 0[0] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217268:217268 [0] NCCL INFO Channel 13/0 : 0[0] -> 1[1] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217278:217278 [6] NCCL INFO Channel 19/0 : 6[6] -> 7[7] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217274:217274 [4] NCCL INFO Channel 21/0 : 4[4] -> 5[5] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217278:217278 [6] NCCL INFO Channel 20/0 : 6[6] -> 7[7] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217269:217269 [1] NCCL INFO Channel 17/0 : 1[1] -> 2[2] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217275:217275 [5] NCCL INFO Channel 13/0 : 5[5] -> 6[6] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217268:217268 [0] NCCL INFO Channel 14/0 : 0[0] -> 1[1] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217279:217279 [7] NCCL INFO Channel 22/0 : 7[7] -> 0[0] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217279:217279 [7] NCCL INFO Channel 23/0 : 7[7] -> 0[0] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217274:217274 [4] NCCL INFO Channel 22/0 : 4[4] -> 5[5] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217269:217269 [1] NCCL INFO Channel 18/0 : 1[1] -> 2[2] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217272:217272 [3] NCCL INFO Channel 16/0 : 3[3] -> 4[4] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217275:217275 [5] NCCL INFO Channel 14/0 : 5[5] -> 6[6] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217268:217268 [0] NCCL INFO Channel 15/0 : 0[0] -> 1[1] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217270:217270 [2] NCCL INFO Channel 16/0 : 2[2] -> 3[3] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217270:217270 [2] NCCL INFO Channel 17/0 : 2[2] -> 3[3] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217272:217272 [3] NCCL INFO Channel 17/0 : 3[3] -> 4[4] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217274:217274 [4] NCCL INFO Channel 23/0 : 4[4] -> 5[5] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217268:217268 [0] NCCL INFO Channel 16/0 : 0[0] -> 1[1] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217275:217275 [5] NCCL INFO Channel 15/0 : 5[5] -> 6[6] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217275:217275 [5] NCCL INFO Channel 16/0 : 5[5] -> 6[6] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217269:217269 [1] NCCL INFO Channel 19/0 : 1[1] -> 2[2] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217278:217278 [6] NCCL INFO Channel 21/0 : 6[6] -> 7[7] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217268:217268 [0] NCCL INFO Channel 17/0 : 0[0] -> 1[1] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217272:217272 [3] NCCL INFO Channel 18/0 : 3[3] -> 4[4] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217272:217272 [3] NCCL INFO Channel 19/0 : 3[3] -> 4[4] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217270:217270 [2] NCCL INFO Channel 18/0 : 2[2] -> 3[3] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217278:217278 [6] NCCL INFO Channel 22/0 : 6[6] -> 7[7] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217268:217268 [0] NCCL INFO Channel 18/0 : 0[0] -> 1[1] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217269:217269 [1] NCCL INFO Channel 20/0 : 1[1] -> 2[2] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217278:217278 [6] NCCL INFO Channel 23/0 : 6[6] -> 7[7] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217268:217268 [0] NCCL INFO Channel 19/0 : 0[0] -> 1[1] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217275:217275 [5] NCCL INFO Channel 17/0 : 5[5] -> 6[6] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217270:217270 [2] NCCL INFO Channel 19/0 : 2[2] -> 3[3] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217272:217272 [3] NCCL INFO Channel 20/0 : 3[3] -> 4[4] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217270:217270 [2] NCCL INFO Channel 20/0 : 2[2] -> 3[3] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217275:217275 [5] NCCL INFO Channel 18/0 : 5[5] -> 6[6] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217268:217268 [0] NCCL INFO Channel 20/0 : 0[0] -> 1[1] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217269:217269 [1] NCCL INFO Channel 21/0 : 1[1] -> 2[2] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217272:217272 [3] NCCL INFO Channel 21/0 : 3[3] -> 4[4] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217270:217270 [2] NCCL INFO Channel 21/0 : 2[2] -> 3[3] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217275:217275 [5] NCCL INFO Channel 19/0 : 5[5] -> 6[6] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217269:217269 [1] NCCL INFO Channel 22/0 : 1[1] -> 2[2] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217272:217272 [3] NCCL INFO Channel 22/0 : 3[3] -> 4[4] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217270:217270 [2] NCCL INFO Channel 22/0 : 2[2] -> 3[3] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217275:217275 [5] NCCL INFO Channel 20/0 : 5[5] -> 6[6] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217268:217268 [0] NCCL INFO Channel 21/0 : 0[0] -> 1[1] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217269:217269 [1] NCCL INFO Channel 23/0 : 1[1] -> 2[2] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217272:217272 [3] NCCL INFO Channel 23/0 : 3[3] -> 4[4] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217270:217270 [2] NCCL INFO Channel 23/0 : 2[2] -> 3[3] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217275:217275 [5] NCCL INFO Channel 21/0 : 5[5] -> 6[6] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217268:217268 [0] NCCL INFO Channel 22/0 : 0[0] -> 1[1] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217268:217268 [0] NCCL INFO Channel 23/0 : 0[0] -> 1[1] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217275:217275 [5] NCCL INFO Channel 22/0 : 5[5] -> 6[6] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217275:217275 [5] NCCL INFO Channel 23/0 : 5[5] -> 6[6] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:217269:217269 [1] NCCL INFO Connected all rings, use ring PXN 0 GDR 1 +ywang29-p4d-debug-1-worker-0:217268:217268 [0] NCCL INFO Connected all rings, use ring PXN 0 GDR 1 +ywang29-p4d-debug-1-worker-0:217270:217270 [2] NCCL INFO Connected all rings, use ring PXN 0 GDR 1 +ywang29-p4d-debug-1-worker-0:217279:217279 [7] NCCL INFO Connected all rings, use ring PXN 0 GDR 1 +ywang29-p4d-debug-1-worker-0:217272:217272 [3] NCCL INFO Connected all rings, use ring PXN 0 GDR 1 +ywang29-p4d-debug-1-worker-0:217275:217275 [5] NCCL INFO Connected all rings, use ring PXN 0 GDR 1 +ywang29-p4d-debug-1-worker-0:217278:217278 [6] NCCL INFO Connected all rings, use ring PXN 0 GDR 1 +ywang29-p4d-debug-1-worker-0:217274:217274 [4] NCCL INFO Connected all rings, use ring PXN 0 GDR 1 +Before initializing optimizer states +MA 4.73 GB Max_MA 4.73 GB CA 4.77 GB Max_CA 5 GB +CPU Virtual Memory: used = 64.87 GB, percent = 5.8% +After initializing optimizer states +MA 4.73 GB Max_MA 5.68 GB CA 5.71 GB Max_CA 6 GB +CPU Virtual Memory: used = 65.1 GB, percent = 5.8% +After initializing ZeRO optimizer +MA 4.73 GB Max_MA 4.73 GB CA 5.71 GB Max_CA 6 GB +CPU Virtual Memory: used = 65.1 GB, percent = 5.8% +[INFO|trainer.py:1587] 2026-04-29 15:40:47,708 >> ***** Running training ***** +[INFO|trainer.py:1588] 2026-04-29 15:40:47,708 >> Num examples = 30,000 +[INFO|trainer.py:1589] 2026-04-29 15:40:47,708 >> Num Epochs = 2 +[INFO|trainer.py:1590] 2026-04-29 15:40:47,708 >> Instantaneous batch size per device = 2 +[INFO|trainer.py:1593] 2026-04-29 15:40:47,708 >> Total train batch size (w. parallel, distributed & accumulation) = 128 +[INFO|trainer.py:1594] 2026-04-29 15:40:47,708 >> Gradient Accumulation steps = 8 +[INFO|trainer.py:1595] 2026-04-29 15:40:47,708 >> Total optimization steps = 470 +[INFO|trainer.py:1596] 2026-04-29 15:40:47,709 >> Number of trainable parameters = 2,031,739,904 +wandb: [wandb.login()] Loaded credentials for https://api.wandb.ai from WANDB_API_KEY. +wandb: Currently logged in as: kkhya (maskmoe) to https://api.wandb.ai. Use `wandb login --relogin` to force relogin +wandb: Tracking run with wandb version 0.26.1 +wandb: Run data is saved locally in /nfs/ywang29/lm-factory/wandb/run-20260429_154048-pzaos2ri +wandb: Run `wandb offline` to turn off syncing. +wandb: Syncing run qwen3_1.7b_coding_fft +wandb: ⭐️ View project at https://wandb.ai/maskmoe/MFT-LM +wandb: 🚀 View run at https://wandb.ai/maskmoe/MFT-LM/runs/pzaos2ri + 0%| | 0/470 [00:00> Saving model checkpoint to saves/qwen3_1.7b/coding/fft/checkpoint-470 +[INFO|configuration_utils.py:432] 2026-04-29 16:06:41,580 >> Configuration saved in saves/qwen3_1.7b/coding/fft/checkpoint-470/config.json +[INFO|configuration_utils.py:803] 2026-04-29 16:06:41,583 >> Configuration saved in saves/qwen3_1.7b/coding/fft/checkpoint-470/generation_config.json + + Writing model shards: 0%| | 0/1 [00:00> Model weights saved in saves/qwen3_1.7b/coding/fft/checkpoint-470/model.safetensors +[INFO|tokenization_utils_base.py:3224] 2026-04-29 16:06:47,792 >> chat template saved in saves/qwen3_1.7b/coding/fft/checkpoint-470/chat_template.jinja +[INFO|tokenization_utils_base.py:2078] 2026-04-29 16:06:47,796 >> tokenizer config file saved in saves/qwen3_1.7b/coding/fft/checkpoint-470/tokenizer_config.json +[INFO|trainer.py:1863] 2026-04-29 16:06:47,957 >> + +Training completed. Do not forget to share your model on huggingface.co/models =) + + + {'train_runtime': '1560', 'train_samples_per_second': '38.45', 'train_steps_per_second': '0.301', 'train_loss': '0.7311', 'epoch': '2'} + 100%|██████████| 470/470 [25:58<00:00, 2.87s/it] 100%|██████████| 470/470 [25:58<00:00, 3.32s/it] +[INFO|trainer.py:3797] 2026-04-29 16:06:49,407 >> Saving model checkpoint to saves/qwen3_1.7b/coding/fft +[INFO|configuration_utils.py:432] 2026-04-29 16:06:49,415 >> Configuration saved in saves/qwen3_1.7b/coding/fft/config.json +[INFO|configuration_utils.py:803] 2026-04-29 16:06:49,423 >> Configuration saved in saves/qwen3_1.7b/coding/fft/generation_config.json + Writing model shards: 0%| | 0/1 [00:00> Model weights saved in saves/qwen3_1.7b/coding/fft/model.safetensors +[INFO|tokenization_utils_base.py:3224] 2026-04-29 16:06:55,648 >> chat template saved in saves/qwen3_1.7b/coding/fft/chat_template.jinja +[INFO|tokenization_utils_base.py:2078] 2026-04-29 16:06:55,651 >> tokenizer config file saved in saves/qwen3_1.7b/coding/fft/tokenizer_config.json +ywang29-p4d-debug-1-worker-0:217278:217278 [6] NCCL INFO comm 0x56515020cac0 rank 6 nranks 8 cudaDev 6 busId a01c0 - Destroy COMPLETE +ywang29-p4d-debug-1-worker-0:217275:217275 [5] NCCL INFO comm 0x55c5481be410 rank 5 nranks 8 cudaDev 5 busId 901d0 - Destroy COMPLETE +ywang29-p4d-debug-1-worker-0:217274:217274 [4] NCCL INFO comm 0x5619aa0bfad0 rank 4 nranks 8 cudaDev 4 busId 901c0 - Destroy COMPLETE +ywang29-p4d-debug-1-worker-0:217272:217272 [3] NCCL INFO comm 0x557ee87f3a90 rank 3 nranks 8 cudaDev 3 busId 201d0 - Destroy COMPLETE +ywang29-p4d-debug-1-worker-0:217270:217270 [2] NCCL INFO comm 0x55febaf71640 rank 2 nranks 8 cudaDev 2 busId 201c0 - Destroy COMPLETE +ywang29-p4d-debug-1-worker-0:217275:217275 [5] NCCL INFO comm 0x55c53c3343a0 rank 5 nranks 8 cudaDev 5 busId 901d0 - Destroy COMPLETE +ywang29-p4d-debug-1-worker-0:217272:217272 [3] NCCL INFO comm 0x557edc8b65e0 rank 3 nranks 8 cudaDev 3 busId 201d0 - Destroy COMPLETE +ywang29-p4d-debug-1-worker-0:217274:217274 [4] NCCL INFO comm 0x56199da84a70 rank 4 nranks 8 cudaDev 4 busId 901c0 - Destroy COMPLETE +[INFO|trainer.py:3797] 2026-04-29 16:06:58,692 >> Saving model checkpoint to saves/qwen3_1.7b/coding/fft +[INFO|configuration_utils.py:432] 2026-04-29 16:06:58,697 >> Configuration saved in saves/qwen3_1.7b/coding/fft/config.json +[INFO|configuration_utils.py:803] 2026-04-29 16:06:58,702 >> Configuration saved in saves/qwen3_1.7b/coding/fft/generation_config.json + Writing model shards: 0%| | 0/1 [00:00> Model weights saved in saves/qwen3_1.7b/coding/fft/model.safetensors +[INFO|tokenization_utils_base.py:3224] 2026-04-29 16:07:05,893 >> chat template saved in saves/qwen3_1.7b/coding/fft/chat_template.jinja +[INFO|tokenization_utils_base.py:2078] 2026-04-29 16:07:05,896 >> tokenizer config file saved in saves/qwen3_1.7b/coding/fft/tokenizer_config.json +[INFO|modelcard.py:266] 2026-04-29 16:07:06,386 >> Dropping the following result as it does not have all the necessary fields: +{'task': {'name': 'Causal Language Modeling', 'type': 'text-generation'}} + Processing Files (0 / 0) : | | 0.00B / 0.00B + New Data Upload : | | 0.00B / 0.00B  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  Processing Files (1 / 1) : 0%| | 11.4MB / 4.07GB, 57.2MB/s + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 0%| | 29.6kB / 4.06GB  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 0%| | 29.6kB / 4.06GB  Processing Files (1 / 2) : 0%| | 11.5MB / 4.07GB, 11.5MB/s + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 0%| | 29.6kB / 4.06GB  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 0%| | 29.6kB / 4.06GB  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 0%| | 636kB / 4.06GB  Processing Files (1 / 2) : 0%| | 12.1MB / 4.07GB, 7.54MB/s + New Data Upload : 0%| | 606kB / 134MB, 379kB/s  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 0%| | 1.24MB / 4.06GB  Processing Files (1 / 2) : 0%| | 12.7MB / 4.07GB, 7.03MB/s + New Data Upload : 1%| | 1.21MB / 134MB, 674kB/s  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 0%| | 8.52MB / 4.06GB  Processing Files (1 / 2) : 0%| | 19.9MB / 4.07GB, 9.97MB/s + New Data Upload : 6%|▋ | 8.49MB / 134MB, 4.25MB/s  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 1%| | 25.5MB / 4.06GB  Processing Files (1 / 2) : 1%| | 36.9MB / 4.07GB, 16.8MB/s + New Data Upload : 19%|█▉ | 25.5MB / 134MB, 11.6MB/s  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 1%| | 39.5MB / 4.06GB  Processing Files (1 / 2) : 1%| | 50.9MB / 4.07GB, 21.2MB/s + New Data Upload : 29%|██▉ | 39.4MB / 134MB, 16.4MB/s  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 1%|▏ | 52.8MB / 4.06GB  Processing Files (1 / 2) : 2%|▏ | 64.2MB / 4.07GB, 24.7MB/s + New Data Upload : 39%|███▉ | 52.8MB / 134MB, 20.3MB/s  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 2%|▏ | 66.7MB / 4.06GB  Processing Files (1 / 2) : 2%|▏ | 78.2MB / 4.07GB, 27.9MB/s + New Data Upload : 50%|████▉ | 66.7MB / 134MB, 23.8MB/s  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 2%|▏ | 66.7MB / 4.06GB  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 2%|▏ | 66.7MB / 4.06GB  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 2%|▏ | 66.7MB / 4.06GB  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 2%|▏ | 67.1MB / 4.06GB  Processing Files (1 / 2) : 2%|▏ | 78.6MB / 4.07GB, 21.8MB/s + New Data Upload : 50%|█████ | 67.1MB / 134MB, 18.6MB/s  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 2%|▏ | 67.7MB / 4.06GB  Processing Files (1 / 2) : 2%|▏ | 79.2MB / 4.07GB, 20.8MB/s + New Data Upload : 34%|███▎ | 67.7MB / 201MB, 17.8MB/s  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 2%|▏ | 74.4MB / 4.06GB  Processing Files (1 / 2) : 2%|▏ | 85.8MB / 4.07GB, 21.5MB/s + New Data Upload : 28%|██▊ | 74.4MB / 268MB, 18.6MB/s  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 2%|▏ | 96.2MB / 4.06GB  Processing Files (1 / 2) : 3%|▎ | 108MB / 4.07GB, 25.6MB/s + New Data Upload : 36%|███▌ | 96.2MB / 268MB, 22.9MB/s  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 3%|▎ | 124MB / 4.06GB  Processing Files (1 / 2) : 3%|▎ | 135MB / 4.07GB, 30.8MB/s + New Data Upload : 37%|███▋ | 124MB / 335MB, 28.2MB/s  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 4%|▎ | 152MB / 4.06GB  Processing Files (1 / 2) : 4%|▍ | 163MB / 4.07GB, 35.5MB/s + New Data Upload : 45%|████▌ | 152MB / 335MB, 33.0MB/s  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 5%|▍ | 186MB / 4.06GB  Processing Files (1 / 2) : 5%|▍ | 197MB / 4.07GB, 41.1MB/s + New Data Upload : 55%|█████▌ | 186MB / 335MB, 38.7MB/s  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 5%|▌ | 211MB / 4.06GB  Processing Files (1 / 2) : 5%|▌ | 222MB / 4.07GB, 44.4MB/s + New Data Upload : 52%|█████▏ | 211MB / 402MB, 42.1MB/s  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 6%|▌ | 232MB / 4.06GB  Processing Files (1 / 2) : 6%|▌ | 243MB / 4.07GB, 46.8MB/s + New Data Upload : 58%|█████▊ | 232MB / 402MB, 44.6MB/s  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 6%|▌ | 254MB / 4.06GB  Processing Files (1 / 2) : 7%|▋ | 265MB / 4.07GB, 49.1MB/s + New Data Upload : 54%|█████▍ | 254MB / 470MB, 47.0MB/s  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 7%|▋ | 282MB / 4.06GB  Processing Files (1 / 2) : 7%|▋ | 294MB / 4.07GB, 52.4MB/s + New Data Upload : 60%|██████ | 282MB / 470MB, 50.4MB/s  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 8%|▊ | 312MB / 4.06GB  Processing Files (1 / 2) : 8%|▊ | 323MB / 4.07GB, 55.7MB/s + New Data Upload : 66%|██████▋ | 312MB / 470MB, 53.7MB/s  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 9%|▊ | 351MB / 4.06GB  Processing Files (1 / 2) : 9%|▉ | 362MB / 4.07GB, 60.4MB/s + New Data Upload : 65%|██████▌ | 351MB / 537MB, 58.5MB/s  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 10%|▉ | 393MB / 4.06GB  Processing Files (1 / 2) : 10%|▉ | 405MB / 4.07GB, 65.3MB/s + New Data Upload : 73%|███████▎ | 393MB / 537MB, 63.4MB/s  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 10%|█ | 420MB / 4.06GB  Processing Files (1 / 2) : 11%|█ | 432MB / 4.07GB, 67.4MB/s + New Data Upload : 78%|███████▊ | 420MB / 537MB, 65.6MB/s  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 11%|█ | 456MB / 4.06GB  Processing Files (1 / 2) : 11%|█▏ | 468MB / 4.07GB, 70.9MB/s + New Data Upload : 76%|███████▌ | 456MB / 604MB, 69.2MB/s  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 12%|█▏ | 489MB / 4.06GB  Processing Files (1 / 2) : 12%|█▏ | 501MB / 4.07GB, 73.7MB/s + New Data Upload : 73%|███████▎ | 489MB / 671MB, 72.0MB/s  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 13%|█▎ | 518MB / 4.06GB  Processing Files (1 / 2) : 13%|█▎ | 530MB / 4.07GB, 75.7MB/s + New Data Upload : 77%|███████▋ | 518MB / 671MB, 74.1MB/s  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 14%|█▎ | 549MB / 4.06GB  Processing Files (1 / 2) : 14%|█▍ | 561MB / 4.07GB, 77.8MB/s + New Data Upload : 74%|███████▍ | 549MB / 738MB, 76.3MB/s  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 14%|█▍ | 575MB / 4.06GB  Processing Files (1 / 2) : 14%|█▍ | 586MB / 4.07GB, 79.2MB/s + New Data Upload : 78%|███████▊ | 575MB / 738MB, 77.7MB/s  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 15%|█▍ | 603MB / 4.06GB  Processing Files (1 / 2) : 15%|█▌ | 615MB / 4.07GB, 80.9MB/s + New Data Upload : 75%|███████▍ | 603MB / 805MB, 79.4MB/s  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 15%|█▌ | 627MB / 4.06GB  Processing Files (1 / 2) : 16%|█▌ | 639MB / 4.07GB, 81.9MB/s + New Data Upload : 78%|███████▊ | 627MB / 805MB, 80.4MB/s  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 16%|█▌ | 656MB / 4.06GB  Processing Files (1 / 2) : 16%|█▋ | 667MB / 4.07GB, 83.4MB/s + New Data Upload : 75%|███████▌ | 656MB / 872MB, 82.0MB/s  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 17%|█▋ | 693MB / 4.06GB  Processing Files (1 / 2) : 17%|█▋ | 705MB / 4.07GB, 85.9MB/s + New Data Upload : 79%|███████▉ | 693MB / 872MB, 84.5MB/s  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 18%|█▊ | 730MB / 4.06GB  Processing Files (1 / 2) : 18%|█▊ | 741MB / 4.07GB, 88.2MB/s + New Data Upload : 78%|███████▊ | 730MB / 939MB, 86.9MB/s  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 19%|█▉ | 775MB / 4.06GB  Processing Files (1 / 2) : 19%|█▉ | 786MB / 4.07GB, 91.4MB/s + New Data Upload : 77%|███████▋ | 775MB / 1.01GB, 90.1MB/s  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 20%|██ | 814MB / 4.06GB  Processing Files (1 / 2) : 20%|██ | 826MB / 4.07GB, 93.8MB/s + New Data Upload : 81%|████████ | 814MB / 1.01GB, 92.5MB/s  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 21%|██ | 852MB / 4.06GB  Processing Files (1 / 2) : 21%|██ | 864MB / 4.07GB, 96.0MB/s + New Data Upload : 79%|███████▉ | 852MB / 1.07GB, 94.7MB/s  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 22%|██▏ | 886MB / 4.06GB  Processing Files (1 / 2) : 22%|██▏ | 897MB / 4.07GB, 97.5MB/s + New Data Upload : 83%|████████▎ | 886MB / 1.07GB, 96.3MB/s  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 23%|██▎ | 926MB / 4.06GB  Processing Files (1 / 2) : 23%|██▎ | 938MB / 4.07GB, 99.7MB/s + New Data Upload : 81%|████████ | 926MB / 1.14GB, 98.5MB/s  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 24%|██▍ | 974MB / 4.06GB  Processing Files (1 / 2) : 24%|██▍ | 986MB / 4.07GB, 103MB/s + New Data Upload : 85%|████████▌ | 974MB / 1.14GB, 101MB/s  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 25%|██▍ | 1.01GB / 4.06GB  Processing Files (1 / 2) : 25%|██▌ | 1.02GB / 4.07GB, 104MB/s + New Data Upload : 89%|████████▉ | 1.01GB / 1.14GB, 103MB/s  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 26%|██▌ | 1.04GB / 4.06GB  Processing Files (1 / 2) : 26%|██▌ | 1.05GB / 4.07GB, 105MB/s + New Data Upload : 91%|█████████▏| 1.04GB / 1.14GB, 104MB/s  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 26%|██▌ | 1.07GB / 4.06GB  Processing Files (1 / 2) : 26%|██▋ | 1.08GB / 4.07GB, 106MB/s + New Data Upload : 94%|█████████▎| 1.07GB / 1.14GB, 105MB/s  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 27%|██▋ | 1.09GB / 4.06GB  Processing Files (1 / 2) : 27%|██▋ | 1.10GB / 4.07GB, 107MB/s + New Data Upload : 95%|█████████▌| 1.09GB / 1.14GB, 107MB/s  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 27%|██▋ | 1.10GB / 4.06GB  Processing Files (1 / 2) : 27%|██▋ | 1.11GB / 4.07GB, 108MB/s + New Data Upload : 97%|█████████▋| 1.10GB / 1.14GB, 108MB/s  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 27%|██▋ | 1.11GB / 4.06GB  Processing Files (1 / 2) : 28%|██▊ | 1.12GB / 4.07GB, 109MB/s + New Data Upload : 98%|█████████▊| 1.11GB / 1.14GB, 109MB/s  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 28%|██▊ | 1.12GB / 4.06GB  Processing Files (1 / 2) : 28%|██▊ | 1.14GB / 4.07GB, 110MB/s + New Data Upload : 99%|█████████▊| 1.12GB / 1.14GB, 110MB/s  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 30%|██▉ | 1.20GB / 4.06GB  Processing Files (1 / 2) : 30%|██▉ | 1.21GB / 4.07GB, 118MB/s + New Data Upload : 100%|█████████▉| 1.14GB / 1.14GB, 112MB/s  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 30%|██▉ | 1.20GB / 4.06GB  Processing Files (1 / 2) : 30%|██▉ | 1.21GB / 4.07GB, 118MB/s + New Data Upload : 94%|█████████▍| 1.14GB / 1.21GB, 112MB/s  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 30%|██▉ | 1.21GB / 4.06GB  Processing Files (1 / 2) : 30%|██▉ | 1.22GB / 4.07GB, 118MB/s + New Data Upload : 95%|█████████▍| 1.14GB / 1.21GB, 112MB/s  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 30%|██▉ | 1.21GB / 4.06GB  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 30%|██▉ | 1.21GB / 4.06GB  Processing Files (1 / 2) : 30%|███ | 1.23GB / 4.07GB, 119MB/s + New Data Upload : 91%|█████████ | 1.15GB / 1.27GB, 113MB/s  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 30%|███ | 1.24GB / 4.06GB  Processing Files (1 / 2) : 31%|███ | 1.25GB / 4.07GB, 120MB/s + New Data Upload : 88%|████████▊ | 1.18GB / 1.34GB, 114MB/s  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 31%|███ | 1.26GB / 4.06GB  Processing Files (1 / 2) : 31%|███ | 1.27GB / 4.07GB, 121MB/s + New Data Upload : 85%|████████▌ | 1.20GB / 1.41GB, 115MB/s  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 32%|███▏ | 1.29GB / 4.06GB  Processing Files (1 / 2) : 32%|███▏ | 1.30GB / 4.07GB, 123MB/s + New Data Upload : 88%|████████▊ | 1.23GB / 1.41GB, 117MB/s  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 33%|███▎ | 1.34GB / 4.06GB  Processing Files (1 / 2) : 33%|███▎ | 1.35GB / 4.07GB, 126MB/s + New Data Upload : 87%|████████▋ | 1.28GB / 1.48GB, 120MB/s  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 34%|███▍ | 1.37GB / 4.06GB  Processing Files (1 / 2) : 34%|███▍ | 1.39GB / 4.07GB, 128MB/s + New Data Upload : 89%|████████▉ | 1.31GB / 1.48GB, 122MB/s  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 34%|███▍ | 1.40GB / 4.06GB  Processing Files (1 / 2) : 35%|███▍ | 1.41GB / 4.07GB, 130MB/s + New Data Upload : 87%|████████▋ | 1.33GB / 1.54GB, 124MB/s  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 35%|███▍ | 1.42GB / 4.06GB  Processing Files (1 / 2) : 35%|███▌ | 1.43GB / 4.07GB, 133MB/s + New Data Upload : 88%|████████▊ | 1.36GB / 1.54GB, 127MB/s  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 36%|███▌ | 1.45GB / 4.06GB  Processing Files (1 / 2) : 36%|███▌ | 1.46GB / 4.07GB, 136MB/s + New Data Upload : 86%|████████▋ | 1.39GB / 1.61GB, 130MB/s  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 36%|███▋ | 1.48GB / 4.06GB  Processing Files (1 / 2) : 37%|███▋ | 1.49GB / 4.07GB, 139MB/s + New Data Upload : 85%|████████▍ | 1.42GB / 1.68GB, 133MB/s  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 38%|███▊ | 1.53GB / 4.06GB  Processing Files (1 / 2) : 38%|███▊ | 1.54GB / 4.07GB, 143MB/s + New Data Upload : 88%|████████▊ | 1.47GB / 1.68GB, 137MB/s  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 39%|███▉ | 1.58GB / 4.06GB  Processing Files (1 / 2) : 39%|███▉ | 1.60GB / 4.07GB, 148MB/s + New Data Upload : 91%|█████████ | 1.52GB / 1.68GB, 142MB/s  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 40%|███▉ | 1.62GB / 4.06GB  Processing Files (1 / 2) : 40%|████ | 1.63GB / 4.07GB, 149MB/s + New Data Upload : 89%|████████▉ | 1.56GB / 1.74GB, 144MB/s  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 41%|████ | 1.66GB / 4.06GB  Processing Files (1 / 2) : 41%|████ | 1.67GB / 4.07GB, 150MB/s + New Data Upload : 91%|█████████▏| 1.60GB / 1.74GB, 144MB/s  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 42%|████▏ | 1.69GB / 4.06GB  Processing Files (1 / 2) : 42%|████▏ | 1.70GB / 4.07GB, 151MB/s + New Data Upload : 90%|████████▉ | 1.63GB / 1.81GB, 145MB/s  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 42%|████▏ | 1.70GB / 4.06GB  Processing Files (1 / 2) : 42%|████▏ | 1.72GB / 4.07GB, 149MB/s + New Data Upload : 91%|█████████ | 1.64GB / 1.81GB, 143MB/s  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 43%|████▎ | 1.73GB / 4.06GB  Processing Files (1 / 2) : 43%|████▎ | 1.74GB / 4.07GB, 149MB/s + New Data Upload : 89%|████████▉ | 1.67GB / 1.88GB, 143MB/s  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 44%|████▎ | 1.77GB / 4.06GB  Processing Files (1 / 2) : 44%|████▎ | 1.78GB / 4.07GB, 151MB/s + New Data Upload : 91%|█████████ | 1.71GB / 1.88GB, 145MB/s  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 45%|████▍ | 1.81GB / 4.06GB  Processing Files (1 / 2) : 45%|████▍ | 1.82GB / 4.07GB, 153MB/s + New Data Upload : 90%|████████▉ | 1.75GB / 1.95GB, 147MB/s  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 46%|████▌ | 1.85GB / 4.06GB  Processing Files (1 / 2) : 46%|████▌ | 1.86GB / 4.07GB, 154MB/s + New Data Upload : 89%|████████▉ | 1.79GB / 2.01GB, 148MB/s  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 46%|████▋ | 1.89GB / 4.06GB  Processing Files (1 / 2) : 47%|████▋ | 1.90GB / 4.07GB, 155MB/s + New Data Upload : 91%|█████████ | 1.83GB / 2.01GB, 149MB/s  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 47%|████▋ | 1.92GB / 4.06GB  Processing Files (1 / 2) : 47%|████▋ | 1.93GB / 4.07GB, 154MB/s + New Data Upload : 93%|█████████▎| 1.86GB / 2.01GB, 148MB/s  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 48%|████▊ | 1.95GB / 4.06GB  Processing Files (1 / 2) : 48%|████▊ | 1.96GB / 4.07GB, 152MB/s + New Data Upload : 91%|█████████ | 1.88GB / 2.08GB, 146MB/s  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 49%|████▊ | 1.98GB / 4.06GB  Processing Files (1 / 2) : 49%|████▉ | 1.99GB / 4.07GB, 153MB/s + New Data Upload : 89%|████████▉ | 1.92GB / 2.15GB, 147MB/s  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 49%|████▉ | 2.01GB / 4.06GB  Processing Files (1 / 2) : 50%|████▉ | 2.02GB / 4.07GB, 152MB/s + New Data Upload : 91%|█████████ | 1.95GB / 2.15GB, 146MB/s  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 51%|█████ | 2.05GB / 4.06GB  Processing Files (1 / 2) : 51%|█████ | 2.07GB / 4.07GB, 153MB/s + New Data Upload : 90%|█████████ | 1.99GB / 2.21GB, 147MB/s  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 52%|█████▏ | 2.10GB / 4.06GB  Processing Files (1 / 2) : 52%|█████▏ | 2.11GB / 4.07GB, 155MB/s + New Data Upload : 92%|█████████▏| 2.04GB / 2.21GB, 149MB/s  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 53%|█████▎ | 2.15GB / 4.06GB  Processing Files (1 / 2) : 53%|█████▎ | 2.16GB / 4.07GB, 157MB/s + New Data Upload : 92%|█████████▏| 2.09GB / 2.28GB, 151MB/s  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 54%|█████▍ | 2.20GB / 4.06GB  Processing Files (1 / 2) : 54%|█████▍ | 2.21GB / 4.07GB, 160MB/s + New Data Upload : 91%|█████████▏| 2.14GB / 2.35GB, 154MB/s  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 55%|█████▌ | 2.24GB / 4.06GB  Processing Files (1 / 2) : 55%|█████▌ | 2.25GB / 4.07GB, 160MB/s + New Data Upload : 93%|█████████▎| 2.17GB / 2.35GB, 154MB/s  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 56%|█████▌ | 2.27GB / 4.06GB  Processing Files (1 / 2) : 56%|█████▌ | 2.28GB / 4.07GB, 161MB/s + New Data Upload : 92%|█████████▏| 2.21GB / 2.41GB, 155MB/s  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 56%|█████▋ | 2.29GB / 4.06GB  Processing Files (1 / 2) : 57%|█████▋ | 2.30GB / 4.07GB, 160MB/s + New Data Upload : 92%|█████████▏| 2.23GB / 2.41GB, 155MB/s  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 57%|█████▋ | 2.32GB / 4.06GB  Processing Files (1 / 2) : 57%|█████▋ | 2.33GB / 4.07GB, 159MB/s + New Data Upload : 91%|█████████ | 2.26GB / 2.48GB, 154MB/s  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 58%|█████▊ | 2.35GB / 4.06GB  Processing Files (1 / 2) : 58%|█████▊ | 2.36GB / 4.07GB, 159MB/s + New Data Upload : 92%|█████████▏| 2.29GB / 2.48GB, 153MB/s  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 58%|█████▊ | 2.38GB / 4.06GB  Processing Files (1 / 2) : 59%|█████▊ | 2.39GB / 4.07GB, 157MB/s + New Data Upload : 91%|█████████ | 2.31GB / 2.55GB, 151MB/s  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 60%|█████▉ | 2.42GB / 4.06GB  Processing Files (1 / 2) : 60%|█████▉ | 2.44GB / 4.07GB, 158MB/s + New Data Upload : 93%|█████████▎| 2.36GB / 2.55GB, 152MB/s  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 61%|██████ | 2.47GB / 4.06GB  Processing Files (1 / 2) : 61%|██████ | 2.48GB / 4.07GB, 158MB/s + New Data Upload : 92%|█████████▏| 2.41GB / 2.62GB, 152MB/s  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 61%|██████▏ | 2.49GB / 4.06GB  Processing Files (1 / 2) : 61%|██████▏ | 2.50GB / 4.07GB, 158MB/s + New Data Upload : 93%|█████████▎| 2.43GB / 2.62GB, 152MB/s  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 62%|██████▏ | 2.53GB / 4.06GB  Processing Files (1 / 2) : 62%|██████▏ | 2.54GB / 4.07GB, 157MB/s + New Data Upload : 92%|█████████▏| 2.47GB / 2.68GB, 151MB/s  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 63%|██████▎ | 2.58GB / 4.06GB  Processing Files (1 / 2) : 64%|██████▎ | 2.59GB / 4.07GB, 157MB/s + New Data Upload : 94%|█████████▍| 2.52GB / 2.68GB, 151MB/s  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 65%|██████▍ | 2.62GB / 4.06GB  Processing Files (1 / 2) : 65%|██████▍ | 2.63GB / 4.07GB, 158MB/s + New Data Upload : 93%|█████████▎| 2.56GB / 2.75GB, 152MB/s  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 65%|██████▌ | 2.65GB / 4.06GB  Processing Files (1 / 2) : 65%|██████▌ | 2.66GB / 4.07GB, 158MB/s + New Data Upload : 94%|█████████▍| 2.59GB / 2.75GB, 152MB/s  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 66%|██████▌ | 2.68GB / 4.06GB  Processing Files (1 / 2) : 66%|██████▌ | 2.70GB / 4.07GB, 159MB/s + New Data Upload : 93%|█████████▎| 2.62GB / 2.82GB, 153MB/s  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 67%|██████▋ | 2.71GB / 4.06GB  Processing Files (1 / 2) : 67%|██████▋ | 2.73GB / 4.07GB, 159MB/s + New Data Upload : 94%|█████████▍| 2.65GB / 2.82GB, 153MB/s  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 67%|██████▋ | 2.74GB / 4.06GB  Processing Files (1 / 2) : 68%|██████▊ | 2.75GB / 4.07GB, 161MB/s + New Data Upload : 93%|█████████▎| 2.68GB / 2.88GB, 155MB/s  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 68%|██████▊ | 2.77GB / 4.06GB  Processing Files (1 / 2) : 68%|██████▊ | 2.78GB / 4.07GB, 162MB/s + New Data Upload : 94%|█████████▍| 2.70GB / 2.88GB, 156MB/s  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 69%|██████▉ | 2.80GB / 4.06GB  Processing Files (1 / 2) : 69%|██████▉ | 2.81GB / 4.07GB, 164MB/s + New Data Upload : 93%|█████████▎| 2.74GB / 2.95GB, 158MB/s  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 70%|███████ | 2.85GB / 4.06GB  Processing Files (1 / 2) : 70%|███████ | 2.86GB / 4.07GB, 161MB/s + New Data Upload : 94%|█████████▍| 2.78GB / 2.95GB, 161MB/s  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 71%|███████ | 2.88GB / 4.06GB  Processing Files (1 / 2) : 71%|███████ | 2.89GB / 4.07GB, 165MB/s + New Data Upload : 94%|█████████▎| 2.82GB / 3.02GB, 165MB/s  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 72%|███████▏ | 2.92GB / 4.06GB  Processing Files (1 / 2) : 72%|███████▏ | 2.93GB / 4.07GB, 168MB/s + New Data Upload : 95%|█████████▍| 2.86GB / 3.02GB, 168MB/s  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 73%|███████▎ | 2.96GB / 4.06GB  Processing Files (1 / 2) : 73%|███████▎ | 2.97GB / 4.07GB, 172MB/s + New Data Upload : 94%|█████████▍| 2.90GB / 3.08GB, 172MB/s  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 74%|███████▎ | 2.99GB / 4.06GB  Processing Files (1 / 2) : 74%|███████▎ | 3.00GB / 4.07GB, 174MB/s + New Data Upload : 93%|█████████▎| 2.93GB / 3.15GB, 174MB/s  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 74%|███████▍ | 3.02GB / 4.06GB  Processing Files (1 / 2) : 74%|███████▍ | 3.03GB / 4.07GB, 175MB/s + New Data Upload : 94%|█████████▍| 2.96GB / 3.15GB, 175MB/s  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 75%|███████▌ | 3.06GB / 4.06GB  Processing Files (1 / 2) : 75%|███████▌ | 3.07GB / 4.07GB, 176MB/s + New Data Upload : 93%|█████████▎| 3.00GB / 3.22GB, 176MB/s  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 76%|███████▌ | 3.09GB / 4.06GB  Processing Files (1 / 2) : 76%|███████▌ | 3.10GB / 4.07GB, 176MB/s + New Data Upload : 94%|█████████▍| 3.03GB / 3.22GB, 176MB/s  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 77%|███████▋ | 3.12GB / 4.06GB  Processing Files (1 / 2) : 77%|███████▋ | 3.13GB / 4.07GB, 174MB/s + New Data Upload : 93%|█████████▎| 3.06GB / 3.29GB, 174MB/s  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 78%|███████▊ | 3.15GB / 4.06GB  Processing Files (1 / 2) : 78%|███████▊ | 3.17GB / 4.07GB, 175MB/s + New Data Upload : 92%|█████████▏| 3.09GB / 3.35GB, 175MB/s  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 79%|███████▉ | 3.20GB / 4.06GB  Processing Files (1 / 2) : 79%|███████▉ | 3.22GB / 4.07GB, 177MB/s + New Data Upload : 94%|█████████▎| 3.14GB / 3.35GB, 177MB/s  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 80%|███████▉ | 3.25GB / 4.06GB  Processing Files (1 / 2) : 80%|████████ | 3.26GB / 4.07GB, 180MB/s + New Data Upload : 95%|█████████▌| 3.19GB / 3.35GB, 180MB/s  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 81%|████████ | 3.28GB / 4.06GB  Processing Files (1 / 2) : 81%|████████ | 3.30GB / 4.07GB, 180MB/s + New Data Upload : 94%|█████████▍| 3.22GB / 3.42GB, 180MB/s  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 82%|████████▏ | 3.33GB / 4.06GB  Processing Files (1 / 2) : 82%|████████▏ | 3.34GB / 4.07GB, 182MB/s + New Data Upload : 96%|█████████▌| 3.27GB / 3.42GB, 182MB/s  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 83%|████████▎ | 3.37GB / 4.06GB  Processing Files (1 / 2) : 83%|████████▎ | 3.38GB / 4.07GB, 181MB/s + New Data Upload : 95%|█████████▍| 3.31GB / 3.49GB, 181MB/s  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 84%|████████▎ | 3.40GB / 4.06GB  Processing Files (1 / 2) : 84%|████████▍ | 3.41GB / 4.07GB, 178MB/s + New Data Upload : 96%|█████████▌| 3.34GB / 3.49GB, 178MB/s  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 84%|████████▍ | 3.42GB / 4.06GB  Processing Files (1 / 2) : 84%|████████▍ | 3.44GB / 4.07GB, 177MB/s + New Data Upload : 95%|█████████▍| 3.36GB / 3.55GB, 177MB/s  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 85%|████████▍ | 3.45GB / 4.06GB  Processing Files (1 / 2) : 85%|████████▍ | 3.46GB / 4.07GB, 176MB/s + New Data Upload : 95%|█████████▌| 3.39GB / 3.55GB, 176MB/s  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 86%|████████▌ | 3.48GB / 4.06GB  Processing Files (1 / 2) : 86%|████████▌ | 3.50GB / 4.07GB, 176MB/s + New Data Upload : 95%|█████████▍| 3.42GB / 3.62GB, 176MB/s  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 86%|████████▋ | 3.51GB / 4.06GB  Processing Files (1 / 2) : 87%|████████▋ | 3.53GB / 4.07GB, 178MB/s + New Data Upload : 94%|█████████▎| 3.45GB / 3.69GB, 178MB/s  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 87%|████████▋ | 3.54GB / 4.06GB  Processing Files (1 / 2) : 87%|████████▋ | 3.56GB / 4.07GB, 178MB/s + New Data Upload : 94%|█████████▍| 3.48GB / 3.69GB, 178MB/s  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 88%|████████▊ | 3.59GB / 4.06GB  Processing Files (1 / 2) : 88%|████████▊ | 3.60GB / 4.07GB, 178MB/s + New Data Upload : 94%|█████████▍| 3.53GB / 3.76GB, 178MB/s  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 90%|████████▉ | 3.64GB / 4.06GB  Processing Files (1 / 2) : 90%|████████▉ | 3.65GB / 4.07GB, 179MB/s + New Data Upload : 94%|█████████▎| 3.58GB / 3.82GB, 179MB/s  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 91%|█████████ | 3.68GB / 4.06GB  Processing Files (1 / 2) : 91%|█████████ | 3.70GB / 4.07GB, 180MB/s + New Data Upload : 95%|█████████▍| 3.62GB / 3.82GB, 180MB/s  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 92%|█████████▏| 3.73GB / 4.06GB  Processing Files (1 / 2) : 92%|█████████▏| 3.74GB / 4.07GB, 180MB/s + New Data Upload : 94%|█████████▍| 3.67GB / 3.89GB, 180MB/s  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 93%|█████████▎| 3.78GB / 4.06GB  Processing Files (1 / 2) : 93%|█████████▎| 3.79GB / 4.07GB, 182MB/s + New Data Upload : 94%|█████████▍| 3.72GB / 3.96GB, 182MB/s  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 94%|█████████▍| 3.82GB / 4.06GB  Processing Files (1 / 2) : 94%|█████████▍| 3.83GB / 4.07GB, 184MB/s + New Data Upload : 95%|█████████▍| 3.76GB / 3.96GB, 184MB/s  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 95%|█████████▍| 3.85GB / 4.06GB  Processing Files (1 / 2) : 95%|█████████▍| 3.87GB / 4.07GB, 184MB/s + New Data Upload : 95%|█████████▍| 3.79GB / 4.00GB, 184MB/s  + + + + ...ing/fft/training_args.bin: 9%|▉ | 658B / 7.18kB  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 96%|█████████▌| 3.89GB / 4.06GB  + + + + ...ing/fft/training_args.bin: 9%|▉ | 658B / 7.18kB  Processing Files (1 / 3) : 96%|█████████▌| 3.91GB / 4.07GB, 185MB/s + New Data Upload : 96%|█████████▌| 3.83GB / 4.00GB, 185MB/s  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 97%|█████████▋| 3.92GB / 4.06GB  + + + + ...ing/fft/training_args.bin: 18%|█▊ | 1.32kB / 7.18kB  Processing Files (1 / 3) : 97%|█████████▋| 3.94GB / 4.07GB, 183MB/s + New Data Upload : 97%|█████████▋| 3.86GB / 4.00GB, 183MB/s  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 97%|█████████▋| 3.96GB / 4.06GB  + + + + ...ing/fft/training_args.bin: 47%|████▋ | 3.38kB / 7.18kB  Processing Files (1 / 3) : 97%|█████████▋| 3.97GB / 4.07GB, 182MB/s + New Data Upload : 97%|█████████▋| 3.90GB / 4.00GB, 182MB/s  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 98%|█████████▊| 3.99GB / 4.06GB  + + + + ...ing/fft/training_args.bin: 79%|███████▊ | 5.64kB / 7.18kB  Processing Files (1 / 3) : 98%|█████████▊| 4.00GB / 4.07GB, 180MB/s + New Data Upload : 98%|█████████▊| 3.93GB / 4.00GB, 180MB/s  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 99%|█████████▉| 4.02GB / 4.06GB  + + + + ...ing/fft/training_args.bin: 99%|█████████▉| 7.15kB / 7.18kB  Processing Files (1 / 3) : 99%|█████████▉| 4.03GB / 4.07GB, 178MB/s + New Data Upload : 99%|█████████▉| 3.96GB / 4.00GB, 178MB/s  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 99%|█████████▉| 4.04GB / 4.06GB  + + + + ...ing/fft/training_args.bin: 99%|█████████▉| 7.15kB / 7.18kB  Processing Files (1 / 3) : 99%|█████████▉| 4.05GB / 4.07GB, 176MB/s + New Data Upload : 99%|█████████▉| 3.97GB / 4.00GB, 176MB/s  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 100%|█████████▉| 4.05GB / 4.06GB  + + + + ...ing/fft/training_args.bin: 99%|█████████▉| 7.15kB / 7.18kB  Processing Files (1 / 3) : 100%|█████████▉| 4.06GB / 4.07GB, 175MB/s + New Data Upload : 100%|█████████▉| 3.99GB / 4.00GB, 175MB/s  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 100%|█████████▉| 4.06GB / 4.06GB  + + + + ...ing/fft/training_args.bin: 99%|█████████▉| 7.15kB / 7.18kB  Processing Files (1 / 3) : 100%|█████████▉| 4.07GB / 4.07GB, 173MB/s + New Data Upload : 100%|█████████▉| 4.00GB / 4.00GB, 173MB/s  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 100%|█████████▉| 4.06GB / 4.06GB  + + + + ...ing/fft/training_args.bin: 100%|██████████| 7.18kB / 7.18kB  Processing Files (2 / 3) : 100%|█████████▉| 4.07GB / 4.07GB, 171MB/s + New Data Upload : 100%|█████████▉| 4.00GB / 4.00GB, 171MB/s  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 100%|█████████▉| 4.06GB / 4.06GB  + + + + ...ing/fft/training_args.bin: 100%|██████████| 7.18kB / 7.18kB  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 100%|█████████▉| 4.06GB / 4.06GB  + + + + ...ing/fft/training_args.bin: 100%|██████████| 7.18kB / 7.18kB  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 100%|█████████▉| 4.06GB / 4.06GB  + + + + ...ing/fft/training_args.bin: 100%|██████████| 7.18kB / 7.18kB  Processing Files (2 / 3) : 100%|█████████▉| 4.07GB / 4.07GB, 161MB/s + New Data Upload : 100%|█████████▉| 4.00GB / 4.00GB, 161MB/s  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 100%|██████████| 4.06GB / 4.06GB  + + + + ...ing/fft/training_args.bin: 100%|██████████| 7.18kB / 7.18kB  Processing Files (3 / 3) : 100%|██████████| 4.07GB / 4.07GB, 156MB/s + New Data Upload : 100%|██████████| 4.00GB / 4.00GB, 156MB/s  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 100%|██████████| 4.06GB / 4.06GB  + + + + ...ing/fft/training_args.bin: 100%|██████████| 7.18kB / 7.18kB  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 100%|██████████| 4.06GB / 4.06GB  + + + + ...ing/fft/training_args.bin: 100%|██████████| 7.18kB / 7.18kB  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 100%|██████████| 4.06GB / 4.06GB  + + + + ...ing/fft/training_args.bin: 100%|██████████| 7.18kB / 7.18kB  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 100%|██████████| 4.06GB / 4.06GB  + + + + ...ing/fft/training_args.bin: 100%|██████████| 7.18kB / 7.18kB  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 100%|██████████| 4.06GB / 4.06GB  + + + + ...ing/fft/training_args.bin: 100%|██████████| 7.18kB / 7.18kB  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 100%|██████████| 4.06GB / 4.06GB  + + + + ...ing/fft/training_args.bin: 100%|██████████| 7.18kB / 7.18kB  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 100%|██████████| 4.06GB / 4.06GB  + + + + ...ing/fft/training_args.bin: 100%|██████████| 7.18kB / 7.18kB  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 100%|██████████| 4.06GB / 4.06GB  + + + + ...ing/fft/training_args.bin: 100%|██████████| 7.18kB / 7.18kB  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 100%|██████████| 4.06GB / 4.06GB  + + + + ...ing/fft/training_args.bin: 100%|██████████| 7.18kB / 7.18kB  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 100%|██████████| 4.06GB / 4.06GB  + + + + ...ing/fft/training_args.bin: 100%|██████████| 7.18kB / 7.18kB  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 100%|██████████| 4.06GB / 4.06GB  + + + + ...ing/fft/training_args.bin: 100%|██████████| 7.18kB / 7.18kB  Processing Files (3 / 3) : 100%|██████████| 4.07GB / 4.07GB, 122MB/s + New Data Upload : 100%|██████████| 4.00GB / 4.00GB, 122MB/s + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB + ...ing/fft/model.safetensors: 100%|██████████| 4.06GB / 4.06GB + ...ing/fft/training_args.bin: 100%|██████████| 7.18kB / 7.18kB +***** train metrics ***** + epoch = 2.0 + total_flos = 371128537GF + train_loss = 0.7311 + train_runtime = 0:26:00.24 + train_samples_per_second = 38.455 + train_steps_per_second = 0.301 +Figure saved at: saves/qwen3_1.7b/coding/fft/training_loss.png +[WARNING|2026-04-29 16:07:51] llamafactory.extras.ploting:149 >> No metric eval_loss to plot. +[WARNING|2026-04-29 16:07:51] llamafactory.extras.ploting:149 >> No metric eval_accuracy to plot. +[INFO|trainer.py:3797] 2026-04-29 16:07:53,193 >> Saving model checkpoint to saves/qwen3_1.7b/coding/fft +[INFO|configuration_utils.py:432] 2026-04-29 16:07:53,200 >> Configuration saved in saves/qwen3_1.7b/coding/fft/config.json +[INFO|configuration_utils.py:803] 2026-04-29 16:07:53,204 >> Configuration saved in saves/qwen3_1.7b/coding/fft/generation_config.json + Writing model shards: 0%| | 0/1 [00:00> Model weights saved in saves/qwen3_1.7b/coding/fft/model.safetensors +[INFO|tokenization_utils_base.py:3224] 2026-04-29 16:07:59,965 >> chat template saved in saves/qwen3_1.7b/coding/fft/chat_template.jinja +[INFO|tokenization_utils_base.py:2078] 2026-04-29 16:07:59,970 >> tokenizer config file saved in saves/qwen3_1.7b/coding/fft/tokenizer_config.json +[INFO|modelcard.py:266] 2026-04-29 16:08:00,302 >> Dropping the following result as it does not have all the necessary fields: +{'task': {'name': 'Causal Language Modeling', 'type': 'text-generation'}} + Processing Files (0 / 0) : | | 0.00B / 0.00B + New Data Upload : | | 0.00B / 0.00B  + + ...ing/fft/training_args.bin: 100%|██████████| 7.18kB / 7.18kB  + + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + + ...ing/fft/model.safetensors: 2%|▏ | 72.0MB / 4.06GB  + + ...ing/fft/training_args.bin: 100%|██████████| 7.18kB / 7.18kB  + + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + + ...ing/fft/model.safetensors: 2%|▏ | 72.0MB / 4.06GB  Processing Files (2 / 3) : 2%|▏ | 83.4MB / 4.07GB, ???B/s + + ...ing/fft/training_args.bin: 100%|██████████| 7.18kB / 7.18kB  + + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + + ...ing/fft/model.safetensors: 5%|▍ | 184MB / 4.06GB  Processing Files (2 / 3) : 5%|▍ | 195MB / 4.07GB, 560MB/s + + ...ing/fft/training_args.bin: 100%|██████████| 7.18kB / 7.18kB  + + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + + ...ing/fft/model.safetensors: 8%|▊ | 320MB / 4.06GB  Processing Files (2 / 3) : 8%|▊ | 331MB / 4.07GB, 621MB/s + + ...ing/fft/training_args.bin: 100%|██████████| 7.18kB / 7.18kB  + + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + + ...ing/fft/model.safetensors: 11%|█ | 456MB / 4.06GB  Processing Files (2 / 3) : 11%|█▏ | 467MB / 4.07GB, 641MB/s + + ...ing/fft/training_args.bin: 100%|██████████| 7.18kB / 7.18kB  + + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + + ...ing/fft/model.safetensors: 15%|█▍ | 600MB / 4.06GB  Processing Files (2 / 3) : 15%|█▌ | 611MB / 4.07GB, 660MB/s + + ...ing/fft/training_args.bin: 100%|██████████| 7.18kB / 7.18kB  + + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + + ...ing/fft/model.safetensors: 18%|█▊ | 728MB / 4.06GB  Processing Files (2 / 3) : 18%|█▊ | 739MB / 4.07GB, 656MB/s + + ...ing/fft/training_args.bin: 100%|██████████| 7.18kB / 7.18kB  + + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + + ...ing/fft/model.safetensors: 21%|██ | 856MB / 4.06GB  Processing Files (2 / 3) : 21%|██▏ | 867MB / 4.07GB, 653MB/s + + ...ing/fft/training_args.bin: 100%|██████████| 7.18kB / 7.18kB  + + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + + ...ing/fft/model.safetensors: 24%|██▍ | 992MB / 4.06GB  Processing Files (2 / 3) : 25%|██▍ | 1.00GB / 4.07GB, 657MB/s + + ...ing/fft/training_args.bin: 100%|██████████| 7.18kB / 7.18kB  + + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + + ...ing/fft/model.safetensors: 28%|██▊ | 1.14GB / 4.06GB  Processing Files (2 / 3) : 28%|██▊ | 1.16GB / 4.07GB, 670MB/s + + ...ing/fft/training_args.bin: 100%|██████████| 7.18kB / 7.18kB  + + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + + ...ing/fft/model.safetensors: 32%|███▏ | 1.31GB / 4.06GB  Processing Files (2 / 3) : 32%|███▏ | 1.32GB / 4.07GB, 689MB/s + + ...ing/fft/training_args.bin: 100%|██████████| 7.18kB / 7.18kB  + + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + + ...ing/fft/model.safetensors: 37%|███▋ | 1.50GB / 4.06GB  Processing Files (2 / 3) : 37%|███▋ | 1.51GB / 4.07GB, 712MB/s + + ...ing/fft/training_args.bin: 100%|██████████| 7.18kB / 7.18kB  + + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + + ...ing/fft/model.safetensors: 41%|████▏ | 1.68GB / 4.06GB  Processing Files (2 / 3) : 42%|████▏ | 1.69GB / 4.07GB, 731MB/s + + ...ing/fft/training_args.bin: 100%|██████████| 7.18kB / 7.18kB  + + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + + ...ing/fft/model.safetensors: 46%|████▌ | 1.86GB / 4.06GB  Processing Files (2 / 3) : 46%|████▌ | 1.88GB / 4.07GB, 747MB/s + + ...ing/fft/training_args.bin: 100%|██████████| 7.18kB / 7.18kB  + + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + + ...ing/fft/model.safetensors: 51%|█████ | 2.06GB / 4.06GB  Processing Files (2 / 3) : 51%|█████ | 2.07GB / 4.07GB, 763MB/s + + ...ing/fft/training_args.bin: 100%|██████████| 7.18kB / 7.18kB  + + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + + ...ing/fft/model.safetensors: 55%|█████▌ | 2.25GB / 4.06GB  Processing Files (2 / 3) : 55%|█████▌ | 2.26GB / 4.07GB, 777MB/s + + ...ing/fft/training_args.bin: 100%|██████████| 7.18kB / 7.18kB  + + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + + ...ing/fft/model.safetensors: 60%|██████ | 2.44GB / 4.06GB  Processing Files (2 / 3) : 60%|██████ | 2.45GB / 4.07GB, 790MB/s + + ...ing/fft/training_args.bin: 100%|██████████| 7.18kB / 7.18kB  + + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + + ...ing/fft/model.safetensors: 64%|██████▍ | 2.61GB / 4.06GB  Processing Files (2 / 3) : 64%|██████▍ | 2.62GB / 4.07GB, 793MB/s + + ...ing/fft/training_args.bin: 100%|██████████| 7.18kB / 7.18kB  + + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + + ...ing/fft/model.safetensors: 69%|██████▉ | 2.80GB / 4.06GB  Processing Files (2 / 3) : 69%|██████▉ | 2.81GB / 4.07GB, 802MB/s + + ...ing/fft/training_args.bin: 100%|██████████| 7.18kB / 7.18kB  + + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + + ...ing/fft/model.safetensors: 73%|███████▎ | 2.97GB / 4.06GB  Processing Files (2 / 3) : 73%|███████▎ | 2.98GB / 4.07GB, 804MB/s + + ...ing/fft/training_args.bin: 100%|██████████| 7.18kB / 7.18kB  + + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + + ...ing/fft/model.safetensors: 78%|███████▊ | 3.16GB / 4.06GB  Processing Files (2 / 3) : 78%|███████▊ | 3.17GB / 4.07GB, 813MB/s + + ...ing/fft/training_args.bin: 100%|██████████| 7.18kB / 7.18kB  + + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + + ...ing/fft/model.safetensors: 82%|████████▏ | 3.34GB / 4.06GB  Processing Files (2 / 3) : 82%|████████▏ | 3.36GB / 4.07GB, 818MB/s + + ...ing/fft/training_args.bin: 100%|██████████| 7.18kB / 7.18kB  + + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + + ...ing/fft/model.safetensors: 87%|████████▋ | 3.54GB / 4.06GB  Processing Files (2 / 3) : 87%|████████▋ | 3.55GB / 4.07GB, 825MB/s + + ...ing/fft/training_args.bin: 100%|██████████| 7.18kB / 7.18kB  + + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + + ...ing/fft/model.safetensors: 92%|█████████▏| 3.73GB / 4.06GB  Processing Files (2 / 3) : 92%|█████████▏| 3.74GB / 4.07GB, 831MB/s + + ...ing/fft/training_args.bin: 100%|██████████| 7.18kB / 7.18kB  + + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + + ...ing/fft/model.safetensors: 97%|█████████▋| 3.93GB / 4.06GB  Processing Files (2 / 3) : 97%|█████████▋| 3.94GB / 4.07GB, 838MB/s + + ...ing/fft/training_args.bin: 100%|██████████| 7.18kB / 7.18kB  + + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + + ...ing/fft/model.safetensors: 100%|██████████| 4.06GB / 4.06GB  Processing Files (3 / 3) : 100%|██████████| 4.07GB / 4.07GB, 832MB/s + + ...ing/fft/training_args.bin: 100%|██████████| 7.18kB / 7.18kB  + + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + + ...ing/fft/model.safetensors: 100%|██████████| 4.06GB / 4.06GB  + + ...ing/fft/training_args.bin: 100%|██████████| 7.18kB / 7.18kB  + + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + + ...ing/fft/model.safetensors: 100%|██████████| 4.06GB / 4.06GB  + + ...ing/fft/training_args.bin: 100%|██████████| 7.18kB / 7.18kB  + + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + + ...ing/fft/model.safetensors: 100%|██████████| 4.06GB / 4.06GB  + + ...ing/fft/training_args.bin: 100%|██████████| 7.18kB / 7.18kB  + + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + + ...ing/fft/model.safetensors: 100%|██████████| 4.06GB / 4.06GB  + + ...ing/fft/training_args.bin: 100%|██████████| 7.18kB / 7.18kB  + + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + + ...ing/fft/model.safetensors: 100%|██████████| 4.06GB / 4.06GB  + + ...ing/fft/training_args.bin: 100%|██████████| 7.18kB / 7.18kB  + + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + + ...ing/fft/model.safetensors: 100%|██████████| 4.06GB / 4.06GB  + + ...ing/fft/training_args.bin: 100%|██████████| 7.18kB / 7.18kB  + + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + + ...ing/fft/model.safetensors: 100%|██████████| 4.06GB / 4.06GB  + + ...ing/fft/training_args.bin: 100%|██████████| 7.18kB / 7.18kB  + + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + + ...ing/fft/model.safetensors: 100%|██████████| 4.06GB / 4.06GB  + + ...ing/fft/training_args.bin: 100%|██████████| 7.18kB / 7.18kB  + + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + + ...ing/fft/model.safetensors: 100%|██████████| 4.06GB / 4.06GB  + + ...ing/fft/training_args.bin: 100%|██████████| 7.18kB / 7.18kB  + + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + + ...ing/fft/model.safetensors: 100%|██████████| 4.06GB / 4.06GB  + + ...ing/fft/training_args.bin: 100%|██████████| 7.18kB / 7.18kB  + + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + + ...ing/fft/model.safetensors: 100%|██████████| 4.06GB / 4.06GB  + + ...ing/fft/training_args.bin: 100%|██████████| 7.18kB / 7.18kB  + + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + + ...ing/fft/model.safetensors: 100%|██████████| 4.06GB / 4.06GB  Processing Files (3 / 3) : 100%|██████████| 4.07GB / 4.07GB, 570MB/s + New Data Upload : | | 0.00B / 0.00B, 0.00B/s + ...ing/fft/training_args.bin: 100%|██████████| 7.18kB / 7.18kB + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB + ...ing/fft/model.safetensors: 100%|██████████| 4.06GB / 4.06GB +ywang29-p4d-debug-1-worker-0:217269:217269 [1] NCCL INFO comm 0x55ef4f96de00 rank 1 nranks 8 cudaDev 1 busId 101d0 - Destroy COMPLETE +ywang29-p4d-debug-1-worker-0:217279:217279 [7] NCCL INFO comm 0x5556e3718090 rank 7 nranks 8 cudaDev 7 busId a01d0 - Destroy COMPLETE +ywang29-p4d-debug-1-worker-0:217270:217270 [2] NCCL INFO comm 0x55feadedf680 rank 2 nranks 8 cudaDev 2 busId 201c0 - Destroy COMPLETE +ywang29-p4d-debug-1-worker-0:217278:217278 [6] NCCL INFO comm 0x56514433a870 rank 6 nranks 8 cudaDev 6 busId a01c0 - Destroy COMPLETE +ywang29-p4d-debug-1-worker-0:217268:217268 [0] NCCL INFO comm 0x562c26858c80 rank 0 nranks 8 cudaDev 0 busId 101c0 - Destroy COMPLETE +ywang29-p4d-debug-1-worker-0:217279:217279 [7] NCCL INFO comm 0x5556d7965790 rank 7 nranks 8 cudaDev 7 busId a01d0 - Destroy COMPLETE +ywang29-p4d-debug-1-worker-0:217269:217269 [1] NCCL INFO comm 0x55ef466ff050 rank 1 nranks 8 cudaDev 1 busId 101d0 - Destroy COMPLETE +ywang29-p4d-debug-1-worker-0:217268:217268 [0] NCCL INFO comm 0x562c18cf2a40 rank 0 nranks 8 cudaDev 0 busId 101c0 - Destroy COMPLETE +wandb: +wandb: 🚀 View run qwen3_1.7b_coding_fft at: https://wandb.ai/maskmoe/MFT-LM/runs/pzaos2ri +wandb: Find logs at: wandb/run-20260429_154048-pzaos2ri/logs +ywang29-p4d-debug-1-worker-0:217278:217278 [6] NCCL INFO ENV/Plugin: Closing env plugin ncclEnvDefault +ywang29-p4d-debug-1-worker-0:217279:217279 [7] NCCL INFO ENV/Plugin: Closing env plugin ncclEnvDefault +ywang29-p4d-debug-1-worker-0:217269:217269 [1] NCCL INFO ENV/Plugin: Closing env plugin ncclEnvDefault +ywang29-p4d-debug-1-worker-0:217270:217270 [2] NCCL INFO ENV/Plugin: Closing env plugin ncclEnvDefault +ywang29-p4d-debug-1-worker-0:217268:217268 [0] NCCL INFO ENV/Plugin: Closing env plugin ncclEnvDefault +==== EXPERIMENT COMPLETED: qwen3_1.7b_coding_fft ==== +Log File: saves/qwen3_1.7b/coding/fft/qwen3_1.7b_coding_fft_20260429_154000.log +Timestamp: 2026-04-29 16:08:26 +===================================== diff --git a/qwen3_1.7b_coding_fft_20260429_170521.log b/qwen3_1.7b_coding_fft_20260429_170521.log new file mode 100644 index 0000000..97487a8 --- /dev/null +++ b/qwen3_1.7b_coding_fft_20260429_170521.log @@ -0,0 +1,2844 @@ +==== STARTING EXPERIMENT: qwen3_1.7b_coding_fft ==== +Log File: saves/qwen3_1.7b/coding/fft/qwen3_1.7b_coding_fft_20260429_170521.log +HF Hub: https://huggingface.co/KKHYA/qwen3-1.7b-fft-coding +Timestamp: 2026-04-29 17:05:21 +===================================== +[INFO|2026-04-29 17:05:36] llamafactory.launcher:144 >> Initializing 8 distributed tasks at: 127.0.0.1:42957 +W0429 17:05:37.808000 243330 site-packages/torch/distributed/run.py:852] +W0429 17:05:37.808000 243330 site-packages/torch/distributed/run.py:852] ***************************************** +W0429 17:05:37.808000 243330 site-packages/torch/distributed/run.py:852] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +W0429 17:05:37.808000 243330 site-packages/torch/distributed/run.py:852] ***************************************** +warmup_ratio is deprecated and will be removed in v5.2. Use `warmup_steps` instead. +warmup_ratio is deprecated and will be removed in v5.2. Use `warmup_steps` instead. +warmup_ratio is deprecated and will be removed in v5.2. Use `warmup_steps` instead. +warmup_ratio is deprecated and will be removed in v5.2. Use `warmup_steps` instead. +warmup_ratio is deprecated and will be removed in v5.2. Use `warmup_steps` instead. +warmup_ratio is deprecated and will be removed in v5.2. Use `warmup_steps` instead. +warmup_ratio is deprecated and will be removed in v5.2. Use `warmup_steps` instead. +warmup_ratio is deprecated and will be removed in v5.2. Use `warmup_steps` instead. +[W429 17:05:52.621446681 ProcessGroupNCCL.cpp:929] Warning: TORCH_NCCL_AVOID_RECORD_STREAMS is the default now, this environment variable is thus deprecated. (function operator()) +[W429 17:05:52.624434293 ProcessGroupNCCL.cpp:929] Warning: TORCH_NCCL_AVOID_RECORD_STREAMS is the default now, this environment variable is thus deprecated. (function operator()) +[W429 17:05:52.640682928 ProcessGroupNCCL.cpp:929] Warning: TORCH_NCCL_AVOID_RECORD_STREAMS is the default now, this environment variable is thus deprecated. (function operator()) +ywang29-p4d-debug-1-worker-0:243411:243411 [0] NCCL INFO ENV/Plugin: Could not find: libnccl-env.so +ywang29-p4d-debug-1-worker-0:243411:243411 [0] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth +ywang29-p4d-debug-1-worker-0:243411:243411 [0] NCCL INFO Bootstrap: Using eth0:10.200.170.96<0> +ywang29-p4d-debug-1-worker-0:243411:243411 [0] NCCL INFO cudaDriverVersion 13000 +[W429 17:05:52.660057146 ProcessGroupNCCL.cpp:929] Warning: TORCH_NCCL_AVOID_RECORD_STREAMS is the default now, this environment variable is thus deprecated. (function operator()) +ywang29-p4d-debug-1-worker-0:243411:243411 [0] NCCL INFO NCCL version 2.28.9+cuda13.0 +[W429 17:05:52.673190851 ProcessGroupNCCL.cpp:929] Warning: TORCH_NCCL_AVOID_RECORD_STREAMS is the default now, this environment variable is thus deprecated. (function operator()) +[W429 17:05:52.676049525 ProcessGroupNCCL.cpp:929] Warning: TORCH_NCCL_AVOID_RECORD_STREAMS is the default now, this environment variable is thus deprecated. (function operator()) +[W429 17:05:52.699482425 ProcessGroupNCCL.cpp:929] Warning: TORCH_NCCL_AVOID_RECORD_STREAMS is the default now, this environment variable is thus deprecated. (function operator()) +[W429 17:05:52.738945900 ProcessGroupNCCL.cpp:929] Warning: TORCH_NCCL_AVOID_RECORD_STREAMS is the default now, this environment variable is thus deprecated. (function operator()) +ywang29-p4d-debug-1-worker-0:243414:243414 [3] NCCL INFO ENV/Plugin: Could not find: libnccl-env.so +ywang29-p4d-debug-1-worker-0:243414:243414 [3] NCCL INFO cudaDriverVersion 13000 +ywang29-p4d-debug-1-worker-0:243414:243414 [3] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth +ywang29-p4d-debug-1-worker-0:243414:243414 [3] NCCL INFO Bootstrap: Using eth0:10.200.170.96<0> +ywang29-p4d-debug-1-worker-0:243414:243414 [3] NCCL INFO NCCL version 2.28.9+cuda13.0 +ywang29-p4d-debug-1-worker-0:243414:243414 [3] NCCL INFO Comm config Blocking set to 1 +ywang29-p4d-debug-1-worker-0:243413:243413 [2] NCCL INFO ENV/Plugin: Could not find: libnccl-env.so +ywang29-p4d-debug-1-worker-0:243413:243413 [2] NCCL INFO cudaDriverVersion 13000 +ywang29-p4d-debug-1-worker-0:243413:243413 [2] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth +ywang29-p4d-debug-1-worker-0:243413:243413 [2] NCCL INFO Bootstrap: Using eth0:10.200.170.96<0> +ywang29-p4d-debug-1-worker-0:243413:243413 [2] NCCL INFO NCCL version 2.28.9+cuda13.0 +ywang29-p4d-debug-1-worker-0:243415:243415 [4] NCCL INFO ENV/Plugin: Could not find: libnccl-env.so +ywang29-p4d-debug-1-worker-0:243415:243415 [4] NCCL INFO cudaDriverVersion 13000 +ywang29-p4d-debug-1-worker-0:243415:243415 [4] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth +ywang29-p4d-debug-1-worker-0:243415:243415 [4] NCCL INFO Bootstrap: Using eth0:10.200.170.96<0> +ywang29-p4d-debug-1-worker-0:243415:243415 [4] NCCL INFO NCCL version 2.28.9+cuda13.0 +ywang29-p4d-debug-1-worker-0:243413:243413 [2] NCCL INFO Comm config Blocking set to 1 +ywang29-p4d-debug-1-worker-0:243415:243415 [4] NCCL INFO Comm config Blocking set to 1 +ywang29-p4d-debug-1-worker-0:243416:243416 [5] NCCL INFO ENV/Plugin: Could not find: libnccl-env.so +ywang29-p4d-debug-1-worker-0:243416:243416 [5] NCCL INFO cudaDriverVersion 13000 +ywang29-p4d-debug-1-worker-0:243417:243417 [6] NCCL INFO ENV/Plugin: Could not find: libnccl-env.so +ywang29-p4d-debug-1-worker-0:243412:243412 [1] NCCL INFO ENV/Plugin: Could not find: libnccl-env.so +ywang29-p4d-debug-1-worker-0:243417:243417 [6] NCCL INFO cudaDriverVersion 13000 +ywang29-p4d-debug-1-worker-0:243412:243412 [1] NCCL INFO cudaDriverVersion 13000 +ywang29-p4d-debug-1-worker-0:243416:243416 [5] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth +ywang29-p4d-debug-1-worker-0:243416:243416 [5] NCCL INFO Bootstrap: Using eth0:10.200.170.96<0> +ywang29-p4d-debug-1-worker-0:243417:243417 [6] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth +ywang29-p4d-debug-1-worker-0:243416:243416 [5] NCCL INFO NCCL version 2.28.9+cuda13.0 +ywang29-p4d-debug-1-worker-0:243412:243412 [1] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth +ywang29-p4d-debug-1-worker-0:243417:243417 [6] NCCL INFO Bootstrap: Using eth0:10.200.170.96<0> +ywang29-p4d-debug-1-worker-0:243417:243417 [6] NCCL INFO NCCL version 2.28.9+cuda13.0 +ywang29-p4d-debug-1-worker-0:243412:243412 [1] NCCL INFO Bootstrap: Using eth0:10.200.170.96<0> +ywang29-p4d-debug-1-worker-0:243412:243412 [1] NCCL INFO NCCL version 2.28.9+cuda13.0 +ywang29-p4d-debug-1-worker-0:243411:243411 [0] NCCL INFO Comm config Blocking set to 1 +ywang29-p4d-debug-1-worker-0:243417:243417 [6] NCCL INFO Comm config Blocking set to 1 +ywang29-p4d-debug-1-worker-0:243416:243416 [5] NCCL INFO Comm config Blocking set to 1 +ywang29-p4d-debug-1-worker-0:243412:243412 [1] NCCL INFO Comm config Blocking set to 1 +ywang29-p4d-debug-1-worker-0:243418:243418 [7] NCCL INFO ENV/Plugin: Could not find: libnccl-env.so +ywang29-p4d-debug-1-worker-0:243418:243418 [7] NCCL INFO cudaDriverVersion 13000 +ywang29-p4d-debug-1-worker-0:243418:243418 [7] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth +ywang29-p4d-debug-1-worker-0:243418:243418 [7] NCCL INFO Bootstrap: Using eth0:10.200.170.96<0> +ywang29-p4d-debug-1-worker-0:243418:243418 [7] NCCL INFO NCCL version 2.28.9+cuda13.0 +ywang29-p4d-debug-1-worker-0:243418:243418 [7] NCCL INFO Comm config Blocking set to 1 +ywang29-p4d-debug-1-worker-0:243414:243414 [3] NCCL INFO NET/Plugin: Loaded net plugin Libfabric (v10) +ywang29-p4d-debug-1-worker-0:243414:243414 [3] NCCL INFO Successfully loaded external network plugin /opt/amazon/ofi-nccl/lib/libnccl-net.so +ywang29-p4d-debug-1-worker-0:243414:243414 [3] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth +ywang29-p4d-debug-1-worker-0:243414:243414 [3] NCCL INFO NET/Socket : Using [0]eth0:10.200.170.96<0> +ywang29-p4d-debug-1-worker-0:243414:243414 [3] NCCL INFO Initialized NET plugin Socket +ywang29-p4d-debug-1-worker-0:243414:243414 [3] NCCL INFO Assigned NET plugin Socket to comm +ywang29-p4d-debug-1-worker-0:243414:243414 [3] NCCL INFO Using network Socket +ywang29-p4d-debug-1-worker-0:243415:243415 [4] NCCL INFO NET/Plugin: Loaded net plugin Libfabric (v10) +ywang29-p4d-debug-1-worker-0:243415:243415 [4] NCCL INFO Successfully loaded external network plugin /opt/amazon/ofi-nccl/lib/libnccl-net.so +ywang29-p4d-debug-1-worker-0:243415:243415 [4] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth +ywang29-p4d-debug-1-worker-0:243415:243415 [4] NCCL INFO NET/Socket : Using [0]eth0:10.200.170.96<0> +ywang29-p4d-debug-1-worker-0:243415:243415 [4] NCCL INFO Initialized NET plugin Socket +ywang29-p4d-debug-1-worker-0:243415:243415 [4] NCCL INFO Assigned NET plugin Socket to comm +ywang29-p4d-debug-1-worker-0:243415:243415 [4] NCCL INFO Using network Socket +ywang29-p4d-debug-1-worker-0:243413:243413 [2] NCCL INFO NET/Plugin: Loaded net plugin Libfabric (v10) +ywang29-p4d-debug-1-worker-0:243413:243413 [2] NCCL INFO Successfully loaded external network plugin /opt/amazon/ofi-nccl/lib/libnccl-net.so +ywang29-p4d-debug-1-worker-0:243413:243413 [2] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth +ywang29-p4d-debug-1-worker-0:243413:243413 [2] NCCL INFO NET/Socket : Using [0]eth0:10.200.170.96<0> +ywang29-p4d-debug-1-worker-0:243413:243413 [2] NCCL INFO Initialized NET plugin Socket +ywang29-p4d-debug-1-worker-0:243413:243413 [2] NCCL INFO Assigned NET plugin Socket to comm +ywang29-p4d-debug-1-worker-0:243413:243413 [2] NCCL INFO Using network Socket +ywang29-p4d-debug-1-worker-0:243417:243417 [6] NCCL INFO NET/Plugin: Loaded net plugin Libfabric (v10) +ywang29-p4d-debug-1-worker-0:243417:243417 [6] NCCL INFO Successfully loaded external network plugin /opt/amazon/ofi-nccl/lib/libnccl-net.so +ywang29-p4d-debug-1-worker-0:243417:243417 [6] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth +ywang29-p4d-debug-1-worker-0:243417:243417 [6] NCCL INFO NET/Socket : Using [0]eth0:10.200.170.96<0> +ywang29-p4d-debug-1-worker-0:243417:243417 [6] NCCL INFO Initialized NET plugin Socket +ywang29-p4d-debug-1-worker-0:243417:243417 [6] NCCL INFO Assigned NET plugin Socket to comm +ywang29-p4d-debug-1-worker-0:243417:243417 [6] NCCL INFO Using network Socket +ywang29-p4d-debug-1-worker-0:243412:243412 [1] NCCL INFO NET/Plugin: Loaded net plugin Libfabric (v10) +ywang29-p4d-debug-1-worker-0:243412:243412 [1] NCCL INFO Successfully loaded external network plugin /opt/amazon/ofi-nccl/lib/libnccl-net.so +ywang29-p4d-debug-1-worker-0:243412:243412 [1] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth +ywang29-p4d-debug-1-worker-0:243412:243412 [1] NCCL INFO NET/Socket : Using [0]eth0:10.200.170.96<0> +ywang29-p4d-debug-1-worker-0:243412:243412 [1] NCCL INFO Initialized NET plugin Socket +ywang29-p4d-debug-1-worker-0:243412:243412 [1] NCCL INFO Assigned NET plugin Socket to comm +ywang29-p4d-debug-1-worker-0:243412:243412 [1] NCCL INFO Using network Socket +ywang29-p4d-debug-1-worker-0:243416:243416 [5] NCCL INFO NET/Plugin: Loaded net plugin Libfabric (v10) +ywang29-p4d-debug-1-worker-0:243416:243416 [5] NCCL INFO Successfully loaded external network plugin /opt/amazon/ofi-nccl/lib/libnccl-net.so +ywang29-p4d-debug-1-worker-0:243416:243416 [5] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth +ywang29-p4d-debug-1-worker-0:243416:243416 [5] NCCL INFO NET/Socket : Using [0]eth0:10.200.170.96<0> +ywang29-p4d-debug-1-worker-0:243416:243416 [5] NCCL INFO Initialized NET plugin Socket +ywang29-p4d-debug-1-worker-0:243416:243416 [5] NCCL INFO Assigned NET plugin Socket to comm +ywang29-p4d-debug-1-worker-0:243416:243416 [5] NCCL INFO Using network Socket +ywang29-p4d-debug-1-worker-0:243411:243411 [0] NCCL INFO NET/Plugin: Loaded net plugin Libfabric (v10) +ywang29-p4d-debug-1-worker-0:243411:243411 [0] NCCL INFO Successfully loaded external network plugin /opt/amazon/ofi-nccl/lib/libnccl-net.so +ywang29-p4d-debug-1-worker-0:243411:243411 [0] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth +ywang29-p4d-debug-1-worker-0:243411:243411 [0] NCCL INFO NET/Socket : Using [0]eth0:10.200.170.96<0> +ywang29-p4d-debug-1-worker-0:243411:243411 [0] NCCL INFO Initialized NET plugin Socket +ywang29-p4d-debug-1-worker-0:243411:243411 [0] NCCL INFO Assigned NET plugin Socket to comm +ywang29-p4d-debug-1-worker-0:243411:243411 [0] NCCL INFO Using network Socket +ywang29-p4d-debug-1-worker-0:243418:243418 [7] NCCL INFO NET/Plugin: Loaded net plugin Libfabric (v10) +ywang29-p4d-debug-1-worker-0:243418:243418 [7] NCCL INFO Successfully loaded external network plugin /opt/amazon/ofi-nccl/lib/libnccl-net.so +ywang29-p4d-debug-1-worker-0:243418:243418 [7] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth +ywang29-p4d-debug-1-worker-0:243418:243418 [7] NCCL INFO NET/Socket : Using [0]eth0:10.200.170.96<0> +ywang29-p4d-debug-1-worker-0:243418:243418 [7] NCCL INFO Initialized NET plugin Socket +ywang29-p4d-debug-1-worker-0:243418:243418 [7] NCCL INFO Assigned NET plugin Socket to comm +ywang29-p4d-debug-1-worker-0:243418:243418 [7] NCCL INFO Using network Socket +ywang29-p4d-debug-1-worker-0:243414:243414 [3] NCCL INFO ncclCommInitRankConfig comm 0x55ae88a9dd90 rank 3 nranks 8 cudaDev 3 nvmlDev 3 busId 201d0 commId 0xa7453980088d05e1 - Init START +ywang29-p4d-debug-1-worker-0:243415:243415 [4] NCCL INFO ncclCommInitRankConfig comm 0x561d46f046a0 rank 4 nranks 8 cudaDev 4 nvmlDev 4 busId 901c0 commId 0xa7453980088d05e1 - Init START +ywang29-p4d-debug-1-worker-0:243413:243413 [2] NCCL INFO ncclCommInitRankConfig comm 0x55b7d7b1b9d0 rank 2 nranks 8 cudaDev 2 nvmlDev 2 busId 201c0 commId 0xa7453980088d05e1 - Init START +ywang29-p4d-debug-1-worker-0:243412:243412 [1] NCCL INFO ncclCommInitRankConfig comm 0x558ed75c7da0 rank 1 nranks 8 cudaDev 1 nvmlDev 1 busId 101d0 commId 0xa7453980088d05e1 - Init START +ywang29-p4d-debug-1-worker-0:243414:243414 [3] NCCL INFO RAS client listening socket at ::1<28028> +ywang29-p4d-debug-1-worker-0:243413:243413 [2] NCCL INFO RAS client listening socket at ::1<28028> +ywang29-p4d-debug-1-worker-0:243417:243417 [6] NCCL INFO ncclCommInitRankConfig comm 0x55b4f5018ae0 rank 6 nranks 8 cudaDev 6 nvmlDev 6 busId a01c0 commId 0xa7453980088d05e1 - Init START +ywang29-p4d-debug-1-worker-0:243416:243416 [5] NCCL INFO ncclCommInitRankConfig comm 0x558fd1beff10 rank 5 nranks 8 cudaDev 5 nvmlDev 5 busId 901d0 commId 0xa7453980088d05e1 - Init START +ywang29-p4d-debug-1-worker-0:243416:243416 [5] NCCL INFO RAS client listening socket at ::1<28028> +ywang29-p4d-debug-1-worker-0:243415:243415 [4] NCCL INFO RAS client listening socket at ::1<28028> +ywang29-p4d-debug-1-worker-0:243411:243411 [0] NCCL INFO ncclCommInitRankConfig comm 0x56416eeca870 rank 0 nranks 8 cudaDev 0 nvmlDev 0 busId 101c0 commId 0xa7453980088d05e1 - Init START +ywang29-p4d-debug-1-worker-0:243412:243412 [1] NCCL INFO RAS client listening socket at ::1<28028> +ywang29-p4d-debug-1-worker-0:243418:243418 [7] NCCL INFO ncclCommInitRankConfig comm 0x55ab97689c00 rank 7 nranks 8 cudaDev 7 nvmlDev 7 busId a01d0 commId 0xa7453980088d05e1 - Init START +ywang29-p4d-debug-1-worker-0:243417:243417 [6] NCCL INFO RAS client listening socket at ::1<28028> +ywang29-p4d-debug-1-worker-0:243418:243418 [7] NCCL INFO RAS client listening socket at ::1<28028> +ywang29-p4d-debug-1-worker-0:243411:243411 [0] NCCL INFO RAS client listening socket at ::1<28028> +ywang29-p4d-debug-1-worker-0:243411:243411 [0] NCCL INFO Bootstrap timings total 0.001876 (create 0.000033, send 0.000066, recv 0.000124, ring 0.000111, delay 0.000001) +ywang29-p4d-debug-1-worker-0:243418:243418 [7] NCCL INFO Bootstrap timings total 0.000689 (create 0.000033, send 0.000071, recv 0.000113, ring 0.000103, delay 0.000001) +ywang29-p4d-debug-1-worker-0:243416:243416 [5] NCCL INFO Bootstrap timings total 0.003558 (create 0.000034, send 0.000066, recv 0.000186, ring 0.002903, delay 0.000001) +ywang29-p4d-debug-1-worker-0:243415:243415 [4] NCCL INFO Bootstrap timings total 0.018024 (create 0.000041, send 0.000080, recv 0.014577, ring 0.002899, delay 0.000001) +ywang29-p4d-debug-1-worker-0:243417:243417 [6] NCCL INFO Bootstrap timings total 0.006715 (create 0.000036, send 0.000073, recv 0.006098, ring 0.000133, delay 0.000001) +ywang29-p4d-debug-1-worker-0:243412:243412 [1] NCCL INFO Bootstrap timings total 0.017532 (create 0.000035, send 0.000062, recv 0.000076, ring 0.001294, delay 0.000001) +ywang29-p4d-debug-1-worker-0:243414:243414 [3] NCCL INFO Bootstrap timings total 0.034926 (create 0.000040, send 0.000182, recv 0.016987, ring 0.017193, delay 0.000001) +ywang29-p4d-debug-1-worker-0:243413:243413 [2] NCCL INFO Bootstrap timings total 0.017917 (create 0.000036, send 0.000064, recv 0.000159, ring 0.016940, delay 0.000001) +ywang29-p4d-debug-1-worker-0:243418:243418 [7] NCCL INFO ncclTopoGetCpuAffinity: Affinity for GPU 7 is 24-47,72-95. (GPU affinity = 24-47,72-95 ; CPU affinity = 0-95). +ywang29-p4d-debug-1-worker-0:243418:243418 [7] NCCL INFO NVLS multicast support is not available on dev 7 (NVLS_NCHANNELS 0) +ywang29-p4d-debug-1-worker-0:243411:243411 [0] NCCL INFO ncclTopoGetCpuAffinity: Affinity for GPU 0 is 0-23,48-71. (GPU affinity = 0-23,48-71 ; CPU affinity = 0-95). +ywang29-p4d-debug-1-worker-0:243411:243411 [0] NCCL INFO NVLS multicast support is not available on dev 0 (NVLS_NCHANNELS 0) +ywang29-p4d-debug-1-worker-0:243412:243412 [1] NCCL INFO ncclTopoGetCpuAffinity: Affinity for GPU 1 is 0-23,48-71. (GPU affinity = 0-23,48-71 ; CPU affinity = 0-95). +ywang29-p4d-debug-1-worker-0:243412:243412 [1] NCCL INFO NVLS multicast support is not available on dev 1 (NVLS_NCHANNELS 0) +ywang29-p4d-debug-1-worker-0:243417:243417 [6] NCCL INFO ncclTopoGetCpuAffinity: Affinity for GPU 6 is 24-47,72-95. (GPU affinity = 24-47,72-95 ; CPU affinity = 0-95). +ywang29-p4d-debug-1-worker-0:243417:243417 [6] NCCL INFO NVLS multicast support is not available on dev 6 (NVLS_NCHANNELS 0) +ywang29-p4d-debug-1-worker-0:243413:243413 [2] NCCL INFO ncclTopoGetCpuAffinity: Affinity for GPU 2 is 0-23,48-71. (GPU affinity = 0-23,48-71 ; CPU affinity = 0-95). +ywang29-p4d-debug-1-worker-0:243413:243413 [2] NCCL INFO NVLS multicast support is not available on dev 2 (NVLS_NCHANNELS 0) +ywang29-p4d-debug-1-worker-0:243416:243416 [5] NCCL INFO ncclTopoGetCpuAffinity: Affinity for GPU 5 is 24-47,72-95. (GPU affinity = 24-47,72-95 ; CPU affinity = 0-95). +ywang29-p4d-debug-1-worker-0:243416:243416 [5] NCCL INFO NVLS multicast support is not available on dev 5 (NVLS_NCHANNELS 0) +ywang29-p4d-debug-1-worker-0:243414:243414 [3] NCCL INFO ncclTopoGetCpuAffinity: Affinity for GPU 3 is 0-23,48-71. (GPU affinity = 0-23,48-71 ; CPU affinity = 0-95). +ywang29-p4d-debug-1-worker-0:243414:243414 [3] NCCL INFO NVLS multicast support is not available on dev 3 (NVLS_NCHANNELS 0) +ywang29-p4d-debug-1-worker-0:243415:243415 [4] NCCL INFO ncclTopoGetCpuAffinity: Affinity for GPU 4 is 24-47,72-95. (GPU affinity = 24-47,72-95 ; CPU affinity = 0-95). +ywang29-p4d-debug-1-worker-0:243415:243415 [4] NCCL INFO NVLS multicast support is not available on dev 4 (NVLS_NCHANNELS 0) +ywang29-p4d-debug-1-worker-0:243417:243417 [6] NCCL INFO comm 0x55b4f5018ae0 rank 6 nRanks 8 nNodes 1 localRanks 8 localRank 6 MNNVL 0 +ywang29-p4d-debug-1-worker-0:243412:243412 [1] NCCL INFO comm 0x558ed75c7da0 rank 1 nRanks 8 nNodes 1 localRanks 8 localRank 1 MNNVL 0 +ywang29-p4d-debug-1-worker-0:243416:243416 [5] NCCL INFO comm 0x558fd1beff10 rank 5 nRanks 8 nNodes 1 localRanks 8 localRank 5 MNNVL 0 +ywang29-p4d-debug-1-worker-0:243418:243418 [7] NCCL INFO comm 0x55ab97689c00 rank 7 nRanks 8 nNodes 1 localRanks 8 localRank 7 MNNVL 0 +ywang29-p4d-debug-1-worker-0:243415:243415 [4] NCCL INFO comm 0x561d46f046a0 rank 4 nRanks 8 nNodes 1 localRanks 8 localRank 4 MNNVL 0 +ywang29-p4d-debug-1-worker-0:243414:243414 [3] NCCL INFO comm 0x55ae88a9dd90 rank 3 nRanks 8 nNodes 1 localRanks 8 localRank 3 MNNVL 0 +ywang29-p4d-debug-1-worker-0:243413:243413 [2] NCCL INFO comm 0x55b7d7b1b9d0 rank 2 nRanks 8 nNodes 1 localRanks 8 localRank 2 MNNVL 0 +ywang29-p4d-debug-1-worker-0:243411:243411 [0] NCCL INFO comm 0x56416eeca870 rank 0 nRanks 8 nNodes 1 localRanks 8 localRank 0 MNNVL 0 +ywang29-p4d-debug-1-worker-0:243411:243411 [0] NCCL INFO Channel 00/24 : 0 1 2 3 4 5 6 7 +ywang29-p4d-debug-1-worker-0:243411:243411 [0] NCCL INFO Channel 01/24 : 0 1 2 3 4 5 6 7 +ywang29-p4d-debug-1-worker-0:243411:243411 [0] NCCL INFO Channel 02/24 : 0 1 2 3 4 5 6 7 +ywang29-p4d-debug-1-worker-0:243411:243411 [0] NCCL INFO Channel 03/24 : 0 1 2 3 4 5 6 7 +ywang29-p4d-debug-1-worker-0:243411:243411 [0] NCCL INFO Channel 04/24 : 0 1 2 3 4 5 6 7 +ywang29-p4d-debug-1-worker-0:243415:243415 [4] NCCL INFO Trees [0] 5/-1/-1->4->3 [1] 5/-1/-1->4->3 [2] 5/-1/-1->4->3 [3] 5/-1/-1->4->3 [4] 5/-1/-1->4->3 [5] 5/-1/-1->4->3 [6] 5/-1/-1->4->3 [7] 5/-1/-1->4->3 [8] 5/-1/-1->4->3 [9] 5/-1/-1->4->3 [10] 5/-1/-1->4->3 [11] 5/-1/-1->4->3 [12] 5/-1/-1->4->3 [13] 5/-1/-1->4->3 [14] 5/-1/-1->4->3 [15] 5/-1/-1->4->3 [16] 5/-1/-1->4->3 [17] 5/-1/-1->4->3 [18] 5/-1/-1->4->3 [19] 5/-1/-1->4->3 [20] 5/-1/-1->4->3 [21] 5/-1/-1->4->3 [22] 5/-1/-1->4->3 [23] 5/-1/-1->4->3 +ywang29-p4d-debug-1-worker-0:243417:243417 [6] NCCL INFO Trees [0] 7/-1/-1->6->5 [1] 7/-1/-1->6->5 [2] 7/-1/-1->6->5 [3] 7/-1/-1->6->5 [4] 7/-1/-1->6->5 [5] 7/-1/-1->6->5 [6] 7/-1/-1->6->5 [7] 7/-1/-1->6->5 [8] 7/-1/-1->6->5 [9] 7/-1/-1->6->5 [10] 7/-1/-1->6->5 [11] 7/-1/-1->6->5 [12] 7/-1/-1->6->5 [13] 7/-1/-1->6->5 [14] 7/-1/-1->6->5 [15] 7/-1/-1->6->5 [16] 7/-1/-1->6->5 [17] 7/-1/-1->6->5 [18] 7/-1/-1->6->5 [19] 7/-1/-1->6->5 [20] 7/-1/-1->6->5 [21] 7/-1/-1->6->5 [22] 7/-1/-1->6->5 [23] 7/-1/-1->6->5 +ywang29-p4d-debug-1-worker-0:243411:243411 [0] NCCL INFO Channel 05/24 : 0 1 2 3 4 5 6 7 +ywang29-p4d-debug-1-worker-0:243416:243416 [5] NCCL INFO Trees [0] 6/-1/-1->5->4 [1] 6/-1/-1->5->4 [2] 6/-1/-1->5->4 [3] 6/-1/-1->5->4 [4] 6/-1/-1->5->4 [5] 6/-1/-1->5->4 [6] 6/-1/-1->5->4 [7] 6/-1/-1->5->4 [8] 6/-1/-1->5->4 [9] 6/-1/-1->5->4 [10] 6/-1/-1->5->4 [11] 6/-1/-1->5->4 [12] 6/-1/-1->5->4 [13] 6/-1/-1->5->4 [14] 6/-1/-1->5->4 [15] 6/-1/-1->5->4 [16] 6/-1/-1->5->4 [17] 6/-1/-1->5->4 [18] 6/-1/-1->5->4 [19] 6/-1/-1->5->4 [20] 6/-1/-1->5->4 [21] 6/-1/-1->5->4 [22] 6/-1/-1->5->4 [23] 6/-1/-1->5->4 +ywang29-p4d-debug-1-worker-0:243415:243415 [4] NCCL INFO P2P Chunksize set to 524288 +ywang29-p4d-debug-1-worker-0:243414:243414 [3] NCCL INFO Trees [0] 4/-1/-1->3->2 [1] 4/-1/-1->3->2 [2] 4/-1/-1->3->2 [3] 4/-1/-1->3->2 [4] 4/-1/-1->3->2 [5] 4/-1/-1->3->2 [6] 4/-1/-1->3->2 [7] 4/-1/-1->3->2 [8] 4/-1/-1->3->2 [9] 4/-1/-1->3->2 [10] 4/-1/-1->3->2 [11] 4/-1/-1->3->2 [12] 4/-1/-1->3->2 [13] 4/-1/-1->3->2 [14] 4/-1/-1->3->2 [15] 4/-1/-1->3->2 [16] 4/-1/-1->3->2 [17] 4/-1/-1->3->2 [18] 4/-1/-1->3->2 [19] 4/-1/-1->3->2 [20] 4/-1/-1->3->2 [21] 4/-1/-1->3->2 [22] 4/-1/-1->3->2 [23] 4/-1/-1->3->2 +ywang29-p4d-debug-1-worker-0:243411:243411 [0] NCCL INFO Channel 06/24 : 0 1 2 3 4 5 6 7 +ywang29-p4d-debug-1-worker-0:243417:243417 [6] NCCL INFO P2P Chunksize set to 524288 +ywang29-p4d-debug-1-worker-0:243418:243418 [7] NCCL INFO Trees [0] -1/-1/-1->7->6 [1] -1/-1/-1->7->6 [2] -1/-1/-1->7->6 [3] -1/-1/-1->7->6 [4] -1/-1/-1->7->6 [5] -1/-1/-1->7->6 [6] -1/-1/-1->7->6 [7] -1/-1/-1->7->6 [8] -1/-1/-1->7->6 [9] -1/-1/-1->7->6 [10] -1/-1/-1->7->6 [11] -1/-1/-1->7->6 [12] -1/-1/-1->7->6 [13] -1/-1/-1->7->6 [14] -1/-1/-1->7->6 [15] -1/-1/-1->7->6 [16] -1/-1/-1->7->6 [17] -1/-1/-1->7->6 [18] -1/-1/-1->7->6 [19] -1/-1/-1->7->6 [20] -1/-1/-1->7->6 [21] -1/-1/-1->7->6 [22] -1/-1/-1->7->6 [23] -1/-1/-1->7->6 +ywang29-p4d-debug-1-worker-0:243416:243416 [5] NCCL INFO P2P Chunksize set to 524288 +ywang29-p4d-debug-1-worker-0:243411:243411 [0] NCCL INFO Channel 07/24 : 0 1 2 3 4 5 6 7 +ywang29-p4d-debug-1-worker-0:243414:243414 [3] NCCL INFO P2P Chunksize set to 524288 +ywang29-p4d-debug-1-worker-0:243411:243411 [0] NCCL INFO Channel 08/24 : 0 1 2 3 4 5 6 7 +ywang29-p4d-debug-1-worker-0:243418:243418 [7] NCCL INFO P2P Chunksize set to 524288 +ywang29-p4d-debug-1-worker-0:243411:243411 [0] NCCL INFO Channel 09/24 : 0 1 2 3 4 5 6 7 +ywang29-p4d-debug-1-worker-0:243411:243411 [0] NCCL INFO Channel 10/24 : 0 1 2 3 4 5 6 7 +ywang29-p4d-debug-1-worker-0:243411:243411 [0] NCCL INFO Channel 11/24 : 0 1 2 3 4 5 6 7 +ywang29-p4d-debug-1-worker-0:243411:243411 [0] NCCL INFO Channel 12/24 : 0 1 2 3 4 5 6 7 +ywang29-p4d-debug-1-worker-0:243411:243411 [0] NCCL INFO Channel 13/24 : 0 1 2 3 4 5 6 7 +ywang29-p4d-debug-1-worker-0:243411:243411 [0] NCCL INFO Channel 14/24 : 0 1 2 3 4 5 6 7 +ywang29-p4d-debug-1-worker-0:243411:243411 [0] NCCL INFO Channel 15/24 : 0 1 2 3 4 5 6 7 +ywang29-p4d-debug-1-worker-0:243411:243411 [0] NCCL INFO Channel 16/24 : 0 1 2 3 4 5 6 7 +ywang29-p4d-debug-1-worker-0:243412:243412 [1] NCCL INFO Trees [0] 2/-1/-1->1->0 [1] 2/-1/-1->1->0 [2] 2/-1/-1->1->0 [3] 2/-1/-1->1->0 [4] 2/-1/-1->1->0 [5] 2/-1/-1->1->0 [6] 2/-1/-1->1->0 [7] 2/-1/-1->1->0 [8] 2/-1/-1->1->0 [9] 2/-1/-1->1->0 [10] 2/-1/-1->1->0 [11] 2/-1/-1->1->0 [12] 2/-1/-1->1->0 [13] 2/-1/-1->1->0 [14] 2/-1/-1->1->0 [15] 2/-1/-1->1->0 [16] 2/-1/-1->1->0 [17] 2/-1/-1->1->0 [18] 2/-1/-1->1->0 [19] 2/-1/-1->1->0 [20] 2/-1/-1->1->0 [21] 2/-1/-1->1->0 [22] 2/-1/-1->1->0 [23] 2/-1/-1->1->0 +ywang29-p4d-debug-1-worker-0:243413:243413 [2] NCCL INFO Trees [0] 3/-1/-1->2->1 [1] 3/-1/-1->2->1 [2] 3/-1/-1->2->1 [3] 3/-1/-1->2->1 [4] 3/-1/-1->2->1 [5] 3/-1/-1->2->1 [6] 3/-1/-1->2->1 [7] 3/-1/-1->2->1 [8] 3/-1/-1->2->1 [9] 3/-1/-1->2->1 [10] 3/-1/-1->2->1 [11] 3/-1/-1->2->1 [12] 3/-1/-1->2->1 [13] 3/-1/-1->2->1 [14] 3/-1/-1->2->1 [15] 3/-1/-1->2->1 [16] 3/-1/-1->2->1 [17] 3/-1/-1->2->1 [18] 3/-1/-1->2->1 [19] 3/-1/-1->2->1 [20] 3/-1/-1->2->1 [21] 3/-1/-1->2->1 [22] 3/-1/-1->2->1 [23] 3/-1/-1->2->1 +ywang29-p4d-debug-1-worker-0:243411:243411 [0] NCCL INFO Channel 17/24 : 0 1 2 3 4 5 6 7 +ywang29-p4d-debug-1-worker-0:243411:243411 [0] NCCL INFO Channel 18/24 : 0 1 2 3 4 5 6 7 +ywang29-p4d-debug-1-worker-0:243411:243411 [0] NCCL INFO Channel 19/24 : 0 1 2 3 4 5 6 7 +ywang29-p4d-debug-1-worker-0:243411:243411 [0] NCCL INFO Channel 20/24 : 0 1 2 3 4 5 6 7 +ywang29-p4d-debug-1-worker-0:243412:243412 [1] NCCL INFO P2P Chunksize set to 524288 +ywang29-p4d-debug-1-worker-0:243413:243413 [2] NCCL INFO P2P Chunksize set to 524288 +ywang29-p4d-debug-1-worker-0:243411:243411 [0] NCCL INFO Channel 21/24 : 0 1 2 3 4 5 6 7 +ywang29-p4d-debug-1-worker-0:243411:243411 [0] NCCL INFO Channel 22/24 : 0 1 2 3 4 5 6 7 +ywang29-p4d-debug-1-worker-0:243411:243411 [0] NCCL INFO Channel 23/24 : 0 1 2 3 4 5 6 7 +ywang29-p4d-debug-1-worker-0:243411:243411 [0] NCCL INFO Trees [0] 1/-1/-1->0->-1 [1] 1/-1/-1->0->-1 [2] 1/-1/-1->0->-1 [3] 1/-1/-1->0->-1 [4] 1/-1/-1->0->-1 [5] 1/-1/-1->0->-1 [6] 1/-1/-1->0->-1 [7] 1/-1/-1->0->-1 [8] 1/-1/-1->0->-1 [9] 1/-1/-1->0->-1 [10] 1/-1/-1->0->-1 [11] 1/-1/-1->0->-1 [12] 1/-1/-1->0->-1 [13] 1/-1/-1->0->-1 [14] 1/-1/-1->0->-1 [15] 1/-1/-1->0->-1 [16] 1/-1/-1->0->-1 [17] 1/-1/-1->0->-1 [18] 1/-1/-1->0->-1 [19] 1/-1/-1->0->-1 [20] 1/-1/-1->0->-1 [21] 1/-1/-1->0->-1 [22] 1/-1/-1->0->-1 [23] 1/-1/-1->0->-1 +ywang29-p4d-debug-1-worker-0:243411:243411 [0] NCCL INFO P2P Chunksize set to 524288 +ywang29-p4d-debug-1-worker-0:243416:243416 [5] NCCL INFO PROFILER/Plugin: Could not find: libnccl-profiler.so +ywang29-p4d-debug-1-worker-0:243416:243416 [5] NCCL INFO Check P2P Type isAllDirectP2p 1 directMode 0 isAllCudaP2p 1 +ywang29-p4d-debug-1-worker-0:243414:243414 [3] NCCL INFO PROFILER/Plugin: Could not find: libnccl-profiler.so +ywang29-p4d-debug-1-worker-0:243414:243414 [3] NCCL INFO Check P2P Type isAllDirectP2p 1 directMode 0 isAllCudaP2p 1 +ywang29-p4d-debug-1-worker-0:243412:243412 [1] NCCL INFO PROFILER/Plugin: Could not find: libnccl-profiler.so +ywang29-p4d-debug-1-worker-0:243412:243412 [1] NCCL INFO Check P2P Type isAllDirectP2p 1 directMode 0 isAllCudaP2p 1 +ywang29-p4d-debug-1-worker-0:243416:243804 [0] NCCL INFO [Proxy Service] Device 5 CPU core 77 +ywang29-p4d-debug-1-worker-0:243416:243805 [0] NCCL INFO [Proxy Service UDS] Device 5 CPU core 87 +ywang29-p4d-debug-1-worker-0:243414:243806 [0] NCCL INFO [Proxy Service] Device 3 CPU core 56 +ywang29-p4d-debug-1-worker-0:243415:243415 [4] NCCL INFO PROFILER/Plugin: Could not find: libnccl-profiler.so +ywang29-p4d-debug-1-worker-0:243414:243807 [0] NCCL INFO [Proxy Service UDS] Device 3 CPU core 9 +ywang29-p4d-debug-1-worker-0:243415:243415 [4] NCCL INFO Check P2P Type isAllDirectP2p 1 directMode 0 isAllCudaP2p 1 +ywang29-p4d-debug-1-worker-0:243412:243808 [0] NCCL INFO [Proxy Service] Device 1 CPU core 3 +ywang29-p4d-debug-1-worker-0:243412:243809 [0] NCCL INFO [Proxy Service UDS] Device 1 CPU core 4 +ywang29-p4d-debug-1-worker-0:243415:243810 [0] NCCL INFO [Proxy Service] Device 4 CPU core 43 +ywang29-p4d-debug-1-worker-0:243415:243811 [0] NCCL INFO [Proxy Service UDS] Device 4 CPU core 47 +ywang29-p4d-debug-1-worker-0:243418:243418 [7] NCCL INFO PROFILER/Plugin: Could not find: libnccl-profiler.so +ywang29-p4d-debug-1-worker-0:243418:243418 [7] NCCL INFO Check P2P Type isAllDirectP2p 1 directMode 0 isAllCudaP2p 1 +ywang29-p4d-debug-1-worker-0:243411:243411 [0] NCCL INFO PROFILER/Plugin: Could not find: libnccl-profiler.so +ywang29-p4d-debug-1-worker-0:243418:243812 [0] NCCL INFO [Proxy Service] Device 7 CPU core 93 +ywang29-p4d-debug-1-worker-0:243411:243411 [0] NCCL INFO Check P2P Type isAllDirectP2p 1 directMode 0 isAllCudaP2p 1 +ywang29-p4d-debug-1-worker-0:243418:243813 [0] NCCL INFO [Proxy Service UDS] Device 7 CPU core 46 +ywang29-p4d-debug-1-worker-0:243411:243814 [0] NCCL INFO [Proxy Service] Device 0 CPU core 23 +ywang29-p4d-debug-1-worker-0:243411:243815 [0] NCCL INFO [Proxy Service UDS] Device 0 CPU core 5 +ywang29-p4d-debug-1-worker-0:243417:243417 [6] NCCL INFO PROFILER/Plugin: Could not find: libnccl-profiler.so +ywang29-p4d-debug-1-worker-0:243417:243417 [6] NCCL INFO Check P2P Type isAllDirectP2p 1 directMode 0 isAllCudaP2p 1 +ywang29-p4d-debug-1-worker-0:243417:243817 [0] NCCL INFO [Proxy Service UDS] Device 6 CPU core 42 +ywang29-p4d-debug-1-worker-0:243417:243816 [0] NCCL INFO [Proxy Service] Device 6 CPU core 89 +ywang29-p4d-debug-1-worker-0:243413:243413 [2] NCCL INFO PROFILER/Plugin: Could not find: libnccl-profiler.so +ywang29-p4d-debug-1-worker-0:243413:243413 [2] NCCL INFO Check P2P Type isAllDirectP2p 1 directMode 0 isAllCudaP2p 1 +ywang29-p4d-debug-1-worker-0:243413:243818 [0] NCCL INFO [Proxy Service] Device 2 CPU core 6 +ywang29-p4d-debug-1-worker-0:243413:243819 [0] NCCL INFO [Proxy Service UDS] Device 2 CPU core 59 +ywang29-p4d-debug-1-worker-0:243417:243417 [6] NCCL INFO TUNER/Plugin: Could not find: libnccl-tuner.so +ywang29-p4d-debug-1-worker-0:243417:243417 [6] NCCL INFO TUNER/Plugin: Using nccl_ofi_tuner (v3) +ywang29-p4d-debug-1-worker-0:243417:243417 [6] NCCL INFO NET/OFI NCCL_OFI_TUNER is not available for platform : p4de.24xlarge, Fall back to NCCL's tuner +ywang29-p4d-debug-1-worker-0:243417:243417 [6] NCCL INFO threadThresholds 8/8/64 | 64/8/64 | 512 | 512 +ywang29-p4d-debug-1-worker-0:243417:243417 [6] NCCL INFO 24 coll channels, 24 collnet channels, 0 nvls channels, 32 p2p channels, 32 p2p channels per peer +ywang29-p4d-debug-1-worker-0:243416:243416 [5] NCCL INFO TUNER/Plugin: Could not find: libnccl-tuner.so +ywang29-p4d-debug-1-worker-0:243416:243416 [5] NCCL INFO TUNER/Plugin: Using nccl_ofi_tuner (v3) +ywang29-p4d-debug-1-worker-0:243416:243416 [5] NCCL INFO NET/OFI NCCL_OFI_TUNER is not available for platform : p4de.24xlarge, Fall back to NCCL's tuner +ywang29-p4d-debug-1-worker-0:243416:243416 [5] NCCL INFO threadThresholds 8/8/64 | 64/8/64 | 512 | 512 +ywang29-p4d-debug-1-worker-0:243416:243416 [5] NCCL INFO 24 coll channels, 24 collnet channels, 0 nvls channels, 32 p2p channels, 32 p2p channels per peer +ywang29-p4d-debug-1-worker-0:243412:243412 [1] NCCL INFO TUNER/Plugin: Could not find: libnccl-tuner.so +ywang29-p4d-debug-1-worker-0:243412:243412 [1] NCCL INFO TUNER/Plugin: Using nccl_ofi_tuner (v3) +ywang29-p4d-debug-1-worker-0:243412:243412 [1] NCCL INFO NET/OFI NCCL_OFI_TUNER is not available for platform : p4de.24xlarge, Fall back to NCCL's tuner +ywang29-p4d-debug-1-worker-0:243412:243412 [1] NCCL INFO threadThresholds 8/8/64 | 64/8/64 | 512 | 512 +ywang29-p4d-debug-1-worker-0:243412:243412 [1] NCCL INFO 24 coll channels, 24 collnet channels, 0 nvls channels, 32 p2p channels, 32 p2p channels per peer +ywang29-p4d-debug-1-worker-0:243414:243414 [3] NCCL INFO TUNER/Plugin: Could not find: libnccl-tuner.so +ywang29-p4d-debug-1-worker-0:243414:243414 [3] NCCL INFO TUNER/Plugin: Using nccl_ofi_tuner (v3) +ywang29-p4d-debug-1-worker-0:243414:243414 [3] NCCL INFO NET/OFI NCCL_OFI_TUNER is not available for platform : p4de.24xlarge, Fall back to NCCL's tuner +ywang29-p4d-debug-1-worker-0:243414:243414 [3] NCCL INFO threadThresholds 8/8/64 | 64/8/64 | 512 | 512 +ywang29-p4d-debug-1-worker-0:243414:243414 [3] NCCL INFO 24 coll channels, 24 collnet channels, 0 nvls channels, 32 p2p channels, 32 p2p channels per peer +ywang29-p4d-debug-1-worker-0:243415:243415 [4] NCCL INFO TUNER/Plugin: Could not find: libnccl-tuner.so +ywang29-p4d-debug-1-worker-0:243415:243415 [4] NCCL INFO TUNER/Plugin: Using nccl_ofi_tuner (v3) +ywang29-p4d-debug-1-worker-0:243415:243415 [4] NCCL INFO NET/OFI NCCL_OFI_TUNER is not available for platform : p4de.24xlarge, Fall back to NCCL's tuner +ywang29-p4d-debug-1-worker-0:243415:243415 [4] NCCL INFO threadThresholds 8/8/64 | 64/8/64 | 512 | 512 +ywang29-p4d-debug-1-worker-0:243415:243415 [4] NCCL INFO 24 coll channels, 24 collnet channels, 0 nvls channels, 32 p2p channels, 32 p2p channels per peer +ywang29-p4d-debug-1-worker-0:243411:243411 [0] NCCL INFO TUNER/Plugin: Could not find: libnccl-tuner.so +ywang29-p4d-debug-1-worker-0:243411:243411 [0] NCCL INFO TUNER/Plugin: Using nccl_ofi_tuner (v3) +ywang29-p4d-debug-1-worker-0:243411:243411 [0] NCCL INFO NET/OFI NCCL_OFI_TUNER is not available for platform : p4de.24xlarge, Fall back to NCCL's tuner +ywang29-p4d-debug-1-worker-0:243411:243411 [0] NCCL INFO threadThresholds 8/8/64 | 64/8/64 | 512 | 512 +ywang29-p4d-debug-1-worker-0:243411:243411 [0] NCCL INFO 24 coll channels, 24 collnet channels, 0 nvls channels, 32 p2p channels, 32 p2p channels per peer +ywang29-p4d-debug-1-worker-0:243411:243411 [0] NCCL INFO CC Off, workFifoBytes 1048576 +ywang29-p4d-debug-1-worker-0:243413:243413 [2] NCCL INFO TUNER/Plugin: Could not find: libnccl-tuner.so +ywang29-p4d-debug-1-worker-0:243413:243413 [2] NCCL INFO TUNER/Plugin: Using nccl_ofi_tuner (v3) +ywang29-p4d-debug-1-worker-0:243418:243418 [7] NCCL INFO TUNER/Plugin: Could not find: libnccl-tuner.so +ywang29-p4d-debug-1-worker-0:243413:243413 [2] NCCL INFO NET/OFI NCCL_OFI_TUNER is not available for platform : p4de.24xlarge, Fall back to NCCL's tuner +ywang29-p4d-debug-1-worker-0:243413:243413 [2] NCCL INFO threadThresholds 8/8/64 | 64/8/64 | 512 | 512 +ywang29-p4d-debug-1-worker-0:243413:243413 [2] NCCL INFO 24 coll channels, 24 collnet channels, 0 nvls channels, 32 p2p channels, 32 p2p channels per peer +ywang29-p4d-debug-1-worker-0:243418:243418 [7] NCCL INFO TUNER/Plugin: Using nccl_ofi_tuner (v3) +ywang29-p4d-debug-1-worker-0:243418:243418 [7] NCCL INFO NET/OFI NCCL_OFI_TUNER is not available for platform : p4de.24xlarge, Fall back to NCCL's tuner +ywang29-p4d-debug-1-worker-0:243418:243418 [7] NCCL INFO threadThresholds 8/8/64 | 64/8/64 | 512 | 512 +ywang29-p4d-debug-1-worker-0:243418:243418 [7] NCCL INFO 24 coll channels, 24 collnet channels, 0 nvls channels, 32 p2p channels, 32 p2p channels per peer +ywang29-p4d-debug-1-worker-0:243413:243413 [2] NCCL INFO ncclCommInitRankConfig comm 0x55b7d7b1b9d0 rank 2 nranks 8 cudaDev 2 nvmlDev 2 busId 201c0 commId 0xa7453980088d05e1 - Init COMPLETE +ywang29-p4d-debug-1-worker-0:243415:243415 [4] NCCL INFO ncclCommInitRankConfig comm 0x561d46f046a0 rank 4 nranks 8 cudaDev 4 nvmlDev 4 busId 901c0 commId 0xa7453980088d05e1 - Init COMPLETE +ywang29-p4d-debug-1-worker-0:243417:243417 [6] NCCL INFO ncclCommInitRankConfig comm 0x55b4f5018ae0 rank 6 nranks 8 cudaDev 6 nvmlDev 6 busId a01c0 commId 0xa7453980088d05e1 - Init COMPLETE +ywang29-p4d-debug-1-worker-0:243415:243415 [4] NCCL INFO Init timings - ncclCommInitRankConfig: rank 4 nranks 8 total 0.50 (kernels 0.18, alloc 0.18, bootstrap 0.02, allgathers 0.00, topo 0.04, graphs 0.00, connections 0.06, rest 0.02) +ywang29-p4d-debug-1-worker-0:243417:243417 [6] NCCL INFO Init timings - ncclCommInitRankConfig: rank 6 nranks 8 total 0.49 (kernels 0.18, alloc 0.19, bootstrap 0.01, allgathers 0.00, topo 0.04, graphs 0.00, connections 0.05, rest 0.03) +ywang29-p4d-debug-1-worker-0:243413:243413 [2] NCCL INFO Init timings - ncclCommInitRankConfig: rank 2 nranks 8 total 0.51 (kernels 0.18, alloc 0.18, bootstrap 0.02, allgathers 0.00, topo 0.04, graphs 0.00, connections 0.06, rest 0.02) +ywang29-p4d-debug-1-worker-0:243411:243411 [0] NCCL INFO ncclCommInitRankConfig comm 0x56416eeca870 rank 0 nranks 8 cudaDev 0 nvmlDev 0 busId 101c0 commId 0xa7453980088d05e1 - Init COMPLETE +ywang29-p4d-debug-1-worker-0:243411:243411 [0] NCCL INFO Init timings - ncclCommInitRankConfig: rank 0 nranks 8 total 0.49 (kernels 0.19, alloc 0.18, bootstrap 0.00, allgathers 0.01, topo 0.03, graphs 0.00, connections 0.06, rest 0.02) +ywang29-p4d-debug-1-worker-0:243414:243414 [3] NCCL INFO ncclCommInitRankConfig comm 0x55ae88a9dd90 rank 3 nranks 8 cudaDev 3 nvmlDev 3 busId 201d0 commId 0xa7453980088d05e1 - Init COMPLETE +ywang29-p4d-debug-1-worker-0:243416:243416 [5] NCCL INFO ncclCommInitRankConfig comm 0x558fd1beff10 rank 5 nranks 8 cudaDev 5 nvmlDev 5 busId 901d0 commId 0xa7453980088d05e1 - Init COMPLETE +ywang29-p4d-debug-1-worker-0:243416:243416 [5] NCCL INFO Init timings - ncclCommInitRankConfig: rank 5 nranks 8 total 0.49 (kernels 0.18, alloc 0.19, bootstrap 0.00, allgathers 0.00, topo 0.04, graphs 0.00, connections 0.05, rest 0.03) +ywang29-p4d-debug-1-worker-0:243418:243418 [7] NCCL INFO ncclCommInitRankConfig comm 0x55ab97689c00 rank 7 nranks 8 cudaDev 7 nvmlDev 7 busId a01d0 commId 0xa7453980088d05e1 - Init COMPLETE +ywang29-p4d-debug-1-worker-0:243418:243418 [7] NCCL INFO Init timings - ncclCommInitRankConfig: rank 7 nranks 8 total 0.49 (kernels 0.19, alloc 0.17, bootstrap 0.00, allgathers 0.01, topo 0.03, graphs 0.00, connections 0.06, rest 0.02) +ywang29-p4d-debug-1-worker-0:243414:243414 [3] NCCL INFO Init timings - ncclCommInitRankConfig: rank 3 nranks 8 total 0.51 (kernels 0.19, alloc 0.17, bootstrap 0.03, allgathers 0.00, topo 0.04, graphs 0.00, connections 0.06, rest 0.02) +ywang29-p4d-debug-1-worker-0:243412:243412 [1] NCCL INFO ncclCommInitRankConfig comm 0x558ed75c7da0 rank 1 nranks 8 cudaDev 1 nvmlDev 1 busId 101d0 commId 0xa7453980088d05e1 - Init COMPLETE +ywang29-p4d-debug-1-worker-0:243412:243412 [1] NCCL INFO Init timings - ncclCommInitRankConfig: rank 1 nranks 8 total 0.49 (kernels 0.18, alloc 0.18, bootstrap 0.02, allgathers 0.01, topo 0.04, graphs 0.00, connections 0.06, rest 0.02) +[INFO|2026-04-29 17:05:53] llamafactory.hparams.parser:505 >> Process rank: 7, world size: 8, device: cuda:7, distributed training: True, compute dtype: torch.bfloat16 +[INFO|2026-04-29 17:05:53] llamafactory.hparams.parser:505 >> Process rank: 6, world size: 8, device: cuda:6, distributed training: True, compute dtype: torch.bfloat16 +[INFO|2026-04-29 17:05:53] llamafactory.hparams.parser:505 >> Process rank: 4, world size: 8, device: cuda:4, distributed training: True, compute dtype: torch.bfloat16 +[INFO|2026-04-29 17:05:53] llamafactory.hparams.parser:505 >> Process rank: 5, world size: 8, device: cuda:5, distributed training: True, compute dtype: torch.bfloat16 +[INFO|2026-04-29 17:05:53] llamafactory.hparams.parser:505 >> Process rank: 3, world size: 8, device: cuda:3, distributed training: True, compute dtype: torch.bfloat16 +[INFO|2026-04-29 17:05:53] llamafactory.hparams.parser:505 >> Process rank: 1, world size: 8, device: cuda:1, distributed training: True, compute dtype: torch.bfloat16 +[INFO|2026-04-29 17:05:53] llamafactory.hparams.parser:505 >> Process rank: 2, world size: 8, device: cuda:2, distributed training: True, compute dtype: torch.bfloat16 +[INFO|2026-04-29 17:05:53] llamafactory.hparams.parser:505 >> Process rank: 0, world size: 8, device: cuda:0, distributed training: True, compute dtype: torch.bfloat16 +[INFO|configuration_utils.py:670] 2026-04-29 17:05:53,802 >> loading configuration file config.json from cache at /root/.cache/huggingface/hub/models--Qwen--Qwen3-1.7B/snapshots/70d244cc86ccca08cf5af4e1e306ecf908b1ad5e/config.json +[INFO|configuration_utils.py:742] 2026-04-29 17:05:53,805 >> Model config Qwen3Config { + "architectures": [ + "Qwen3ForCausalLM" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "bos_token_id": 151643, + "dtype": "bfloat16", + "eos_token_id": 151645, + "head_dim": 128, + "hidden_act": "silu", + "hidden_size": 2048, + "initializer_range": 0.02, + "intermediate_size": 6144, + "layer_types": [ + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention" + ], + "masked_layers": null, + "max_position_embeddings": 40960, + "max_window_layers": 28, + "model_type": "qwen3", + "num_attention_heads": 16, + "num_hidden_layers": 28, + "num_key_value_heads": 8, + "pad_token_id": null, + "rms_norm_eps": 1e-06, + "rope_parameters": { + "rope_theta": 1000000, + "rope_type": "default" + }, + "sliding_window": null, + "sparsity_attn": null, + "sparsity_mlp": null, + "subnet_mode": null, + "subnet_type": null, + "threshold_attn": null, + "threshold_mlp": null, + "tie_word_embeddings": true, + "transformers_version": "5.2.0", + "use_cache": true, + "use_sliding_window": false, + "vocab_size": 151936 +} + +[INFO|configuration_utils.py:670] 2026-04-29 17:05:56,244 >> loading configuration file config.json from cache at /root/.cache/huggingface/hub/models--Qwen--Qwen3-1.7B/snapshots/70d244cc86ccca08cf5af4e1e306ecf908b1ad5e/config.json +[INFO|configuration_utils.py:742] 2026-04-29 17:05:56,245 >> Model config Qwen3Config { + "architectures": [ + "Qwen3ForCausalLM" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "bos_token_id": 151643, + "dtype": "bfloat16", + "eos_token_id": 151645, + "head_dim": 128, + "hidden_act": "silu", + "hidden_size": 2048, + "initializer_range": 0.02, + "intermediate_size": 6144, + "layer_types": [ + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention" + ], + "masked_layers": null, + "max_position_embeddings": 40960, + "max_window_layers": 28, + "model_type": "qwen3", + "num_attention_heads": 16, + "num_hidden_layers": 28, + "num_key_value_heads": 8, + "pad_token_id": null, + "rms_norm_eps": 1e-06, + "rope_parameters": { + "rope_theta": 1000000, + "rope_type": "default" + }, + "sliding_window": null, + "sparsity_attn": null, + "sparsity_mlp": null, + "subnet_mode": null, + "subnet_type": null, + "threshold_attn": null, + "threshold_mlp": null, + "tie_word_embeddings": true, + "transformers_version": "5.2.0", + "use_cache": true, + "use_sliding_window": false, + "vocab_size": 151936 +} + +[INFO|configuration_utils.py:670] 2026-04-29 17:05:56,358 >> loading configuration file config.json from cache at /root/.cache/huggingface/hub/models--Qwen--Qwen3-1.7B/snapshots/70d244cc86ccca08cf5af4e1e306ecf908b1ad5e/config.json +[INFO|configuration_utils.py:742] 2026-04-29 17:05:56,359 >> Model config Qwen3Config { + "architectures": [ + "Qwen3ForCausalLM" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "bos_token_id": 151643, + "dtype": "bfloat16", + "eos_token_id": 151645, + "head_dim": 128, + "hidden_act": "silu", + "hidden_size": 2048, + "initializer_range": 0.02, + "intermediate_size": 6144, + "layer_types": [ + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention" + ], + "masked_layers": null, + "max_position_embeddings": 40960, + "max_window_layers": 28, + "model_type": "qwen3", + "num_attention_heads": 16, + "num_hidden_layers": 28, + "num_key_value_heads": 8, + "pad_token_id": null, + "rms_norm_eps": 1e-06, + "rope_parameters": { + "rope_theta": 1000000, + "rope_type": "default" + }, + "sliding_window": null, + "sparsity_attn": null, + "sparsity_mlp": null, + "subnet_mode": null, + "subnet_type": null, + "threshold_attn": null, + "threshold_mlp": null, + "tie_word_embeddings": true, + "transformers_version": "5.2.0", + "use_cache": true, + "use_sliding_window": false, + "vocab_size": 151936 +} + +ywang29-p4d-debug-1-worker-0:243414:243414 [3] NCCL INFO Channel 00/0 : 3[3] -> 4[4] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243414:243414 [3] NCCL INFO Channel 01/0 : 3[3] -> 4[4] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243414:243414 [3] NCCL INFO Channel 02/0 : 3[3] -> 4[4] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243414:243414 [3] NCCL INFO Channel 03/0 : 3[3] -> 4[4] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243414:243414 [3] NCCL INFO Channel 04/0 : 3[3] -> 4[4] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243414:243414 [3] NCCL INFO Channel 05/0 : 3[3] -> 4[4] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243414:243414 [3] NCCL INFO Channel 06/0 : 3[3] -> 4[4] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243414:243414 [3] NCCL INFO Channel 07/0 : 3[3] -> 4[4] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243414:243414 [3] NCCL INFO Channel 08/0 : 3[3] -> 4[4] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243414:243414 [3] NCCL INFO Channel 09/0 : 3[3] -> 4[4] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243414:243414 [3] NCCL INFO Channel 10/0 : 3[3] -> 4[4] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243414:243414 [3] NCCL INFO Channel 11/0 : 3[3] -> 4[4] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243414:243414 [3] NCCL INFO Channel 12/0 : 3[3] -> 4[4] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243414:243414 [3] NCCL INFO Channel 13/0 : 3[3] -> 4[4] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243414:243414 [3] NCCL INFO Channel 14/0 : 3[3] -> 4[4] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243414:243414 [3] NCCL INFO Channel 15/0 : 3[3] -> 4[4] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243414:243414 [3] NCCL INFO Channel 16/0 : 3[3] -> 4[4] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243414:243414 [3] NCCL INFO Channel 17/0 : 3[3] -> 4[4] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243414:243414 [3] NCCL INFO Channel 18/0 : 3[3] -> 4[4] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243414:243414 [3] NCCL INFO Channel 19/0 : 3[3] -> 4[4] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243414:243414 [3] NCCL INFO Channel 20/0 : 3[3] -> 4[4] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243414:243414 [3] NCCL INFO Channel 21/0 : 3[3] -> 4[4] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243414:243414 [3] NCCL INFO Channel 22/0 : 3[3] -> 4[4] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243414:243414 [3] NCCL INFO Channel 23/0 : 3[3] -> 4[4] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243417:243417 [6] NCCL INFO Channel 00/0 : 6[6] -> 7[7] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243417:243417 [6] NCCL INFO Channel 01/0 : 6[6] -> 7[7] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243417:243417 [6] NCCL INFO Channel 02/0 : 6[6] -> 7[7] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243417:243417 [6] NCCL INFO Channel 03/0 : 6[6] -> 7[7] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243417:243417 [6] NCCL INFO Channel 04/0 : 6[6] -> 7[7] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243417:243417 [6] NCCL INFO Channel 05/0 : 6[6] -> 7[7] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243417:243417 [6] NCCL INFO Channel 06/0 : 6[6] -> 7[7] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243417:243417 [6] NCCL INFO Channel 07/0 : 6[6] -> 7[7] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243417:243417 [6] NCCL INFO Channel 08/0 : 6[6] -> 7[7] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243417:243417 [6] NCCL INFO Channel 09/0 : 6[6] -> 7[7] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243417:243417 [6] NCCL INFO Channel 10/0 : 6[6] -> 7[7] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243417:243417 [6] NCCL INFO Channel 11/0 : 6[6] -> 7[7] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243417:243417 [6] NCCL INFO Channel 12/0 : 6[6] -> 7[7] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243417:243417 [6] NCCL INFO Channel 13/0 : 6[6] -> 7[7] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243417:243417 [6] NCCL INFO Channel 14/0 : 6[6] -> 7[7] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243417:243417 [6] NCCL INFO Channel 15/0 : 6[6] -> 7[7] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243417:243417 [6] NCCL INFO Channel 16/0 : 6[6] -> 7[7] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243417:243417 [6] NCCL INFO Channel 17/0 : 6[6] -> 7[7] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243417:243417 [6] NCCL INFO Channel 18/0 : 6[6] -> 7[7] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243417:243417 [6] NCCL INFO Channel 19/0 : 6[6] -> 7[7] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243417:243417 [6] NCCL INFO Channel 20/0 : 6[6] -> 7[7] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243417:243417 [6] NCCL INFO Channel 21/0 : 6[6] -> 7[7] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243417:243417 [6] NCCL INFO Channel 22/0 : 6[6] -> 7[7] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243417:243417 [6] NCCL INFO Channel 23/0 : 6[6] -> 7[7] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243413:243413 [2] NCCL INFO Channel 00/0 : 2[2] -> 3[3] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243413:243413 [2] NCCL INFO Channel 01/0 : 2[2] -> 3[3] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243413:243413 [2] NCCL INFO Channel 02/0 : 2[2] -> 3[3] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243413:243413 [2] NCCL INFO Channel 03/0 : 2[2] -> 3[3] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243413:243413 [2] NCCL INFO Channel 04/0 : 2[2] -> 3[3] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243413:243413 [2] NCCL INFO Channel 05/0 : 2[2] -> 3[3] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243413:243413 [2] NCCL INFO Channel 06/0 : 2[2] -> 3[3] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243413:243413 [2] NCCL INFO Channel 07/0 : 2[2] -> 3[3] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243413:243413 [2] NCCL INFO Channel 08/0 : 2[2] -> 3[3] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243413:243413 [2] NCCL INFO Channel 09/0 : 2[2] -> 3[3] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243413:243413 [2] NCCL INFO Channel 10/0 : 2[2] -> 3[3] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243413:243413 [2] NCCL INFO Channel 11/0 : 2[2] -> 3[3] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243413:243413 [2] NCCL INFO Channel 12/0 : 2[2] -> 3[3] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243413:243413 [2] NCCL INFO Channel 13/0 : 2[2] -> 3[3] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243413:243413 [2] NCCL INFO Channel 14/0 : 2[2] -> 3[3] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243413:243413 [2] NCCL INFO Channel 15/0 : 2[2] -> 3[3] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243413:243413 [2] NCCL INFO Channel 16/0 : 2[2] -> 3[3] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243413:243413 [2] NCCL INFO Channel 17/0 : 2[2] -> 3[3] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243413:243413 [2] NCCL INFO Channel 18/0 : 2[2] -> 3[3] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243413:243413 [2] NCCL INFO Channel 19/0 : 2[2] -> 3[3] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243413:243413 [2] NCCL INFO Channel 20/0 : 2[2] -> 3[3] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243413:243413 [2] NCCL INFO Channel 21/0 : 2[2] -> 3[3] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243413:243413 [2] NCCL INFO Channel 22/0 : 2[2] -> 3[3] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243413:243413 [2] NCCL INFO Channel 23/0 : 2[2] -> 3[3] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243415:243415 [4] NCCL INFO Channel 00/0 : 4[4] -> 5[5] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243415:243415 [4] NCCL INFO Channel 01/0 : 4[4] -> 5[5] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243415:243415 [4] NCCL INFO Channel 02/0 : 4[4] -> 5[5] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243415:243415 [4] NCCL INFO Channel 03/0 : 4[4] -> 5[5] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243416:243416 [5] NCCL INFO Channel 00/0 : 5[5] -> 6[6] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243416:243416 [5] NCCL INFO Channel 01/0 : 5[5] -> 6[6] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243415:243415 [4] NCCL INFO Channel 04/0 : 4[4] -> 5[5] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243415:243415 [4] NCCL INFO Channel 05/0 : 4[4] -> 5[5] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243416:243416 [5] NCCL INFO Channel 02/0 : 5[5] -> 6[6] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243415:243415 [4] NCCL INFO Channel 06/0 : 4[4] -> 5[5] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243416:243416 [5] NCCL INFO Channel 03/0 : 5[5] -> 6[6] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243415:243415 [4] NCCL INFO Channel 07/0 : 4[4] -> 5[5] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243416:243416 [5] NCCL INFO Channel 04/0 : 5[5] -> 6[6] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243415:243415 [4] NCCL INFO Channel 08/0 : 4[4] -> 5[5] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243416:243416 [5] NCCL INFO Channel 05/0 : 5[5] -> 6[6] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243416:243416 [5] NCCL INFO Channel 06/0 : 5[5] -> 6[6] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243415:243415 [4] NCCL INFO Channel 09/0 : 4[4] -> 5[5] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243415:243415 [4] NCCL INFO Channel 10/0 : 4[4] -> 5[5] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243416:243416 [5] NCCL INFO Channel 07/0 : 5[5] -> 6[6] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243415:243415 [4] NCCL INFO Channel 11/0 : 4[4] -> 5[5] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243416:243416 [5] NCCL INFO Channel 08/0 : 5[5] -> 6[6] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243415:243415 [4] NCCL INFO Channel 12/0 : 4[4] -> 5[5] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243416:243416 [5] NCCL INFO Channel 09/0 : 5[5] -> 6[6] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243416:243416 [5] NCCL INFO Channel 10/0 : 5[5] -> 6[6] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243415:243415 [4] NCCL INFO Channel 13/0 : 4[4] -> 5[5] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243415:243415 [4] NCCL INFO Channel 14/0 : 4[4] -> 5[5] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243416:243416 [5] NCCL INFO Channel 11/0 : 5[5] -> 6[6] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243416:243416 [5] NCCL INFO Channel 12/0 : 5[5] -> 6[6] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243415:243415 [4] NCCL INFO Channel 15/0 : 4[4] -> 5[5] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243415:243415 [4] NCCL INFO Channel 16/0 : 4[4] -> 5[5] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243416:243416 [5] NCCL INFO Channel 13/0 : 5[5] -> 6[6] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243416:243416 [5] NCCL INFO Channel 14/0 : 5[5] -> 6[6] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243415:243415 [4] NCCL INFO Channel 17/0 : 4[4] -> 5[5] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243416:243416 [5] NCCL INFO Channel 15/0 : 5[5] -> 6[6] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243415:243415 [4] NCCL INFO Channel 18/0 : 4[4] -> 5[5] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243415:243415 [4] NCCL INFO Channel 19/0 : 4[4] -> 5[5] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243416:243416 [5] NCCL INFO Channel 16/0 : 5[5] -> 6[6] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243415:243415 [4] NCCL INFO Channel 20/0 : 4[4] -> 5[5] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243416:243416 [5] NCCL INFO Channel 17/0 : 5[5] -> 6[6] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243415:243415 [4] NCCL INFO Channel 21/0 : 4[4] -> 5[5] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243416:243416 [5] NCCL INFO Channel 18/0 : 5[5] -> 6[6] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243415:243415 [4] NCCL INFO Channel 22/0 : 4[4] -> 5[5] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243416:243416 [5] NCCL INFO Channel 19/0 : 5[5] -> 6[6] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243415:243415 [4] NCCL INFO Channel 23/0 : 4[4] -> 5[5] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243416:243416 [5] NCCL INFO Channel 20/0 : 5[5] -> 6[6] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243416:243416 [5] NCCL INFO Channel 21/0 : 5[5] -> 6[6] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243416:243416 [5] NCCL INFO Channel 22/0 : 5[5] -> 6[6] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243416:243416 [5] NCCL INFO Channel 23/0 : 5[5] -> 6[6] via P2P/CUMEM/read +[INFO|2026-04-29 17:05:57] llamafactory.data.loader:144 >> Loading dataset allenai/tulu-3-sft-personas-code... +ywang29-p4d-debug-1-worker-0:243412:243412 [1] NCCL INFO Channel 00/0 : 1[1] -> 2[2] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243412:243412 [1] NCCL INFO Channel 01/0 : 1[1] -> 2[2] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243412:243412 [1] NCCL INFO Channel 02/0 : 1[1] -> 2[2] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243412:243412 [1] NCCL INFO Channel 03/0 : 1[1] -> 2[2] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243412:243412 [1] NCCL INFO Channel 04/0 : 1[1] -> 2[2] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243412:243412 [1] NCCL INFO Channel 05/0 : 1[1] -> 2[2] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243412:243412 [1] NCCL INFO Channel 06/0 : 1[1] -> 2[2] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243412:243412 [1] NCCL INFO Channel 07/0 : 1[1] -> 2[2] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243412:243412 [1] NCCL INFO Channel 08/0 : 1[1] -> 2[2] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243412:243412 [1] NCCL INFO Channel 09/0 : 1[1] -> 2[2] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243412:243412 [1] NCCL INFO Channel 10/0 : 1[1] -> 2[2] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243412:243412 [1] NCCL INFO Channel 11/0 : 1[1] -> 2[2] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243412:243412 [1] NCCL INFO Channel 12/0 : 1[1] -> 2[2] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243412:243412 [1] NCCL INFO Channel 13/0 : 1[1] -> 2[2] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243412:243412 [1] NCCL INFO Channel 14/0 : 1[1] -> 2[2] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243412:243412 [1] NCCL INFO Channel 15/0 : 1[1] -> 2[2] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243412:243412 [1] NCCL INFO Channel 16/0 : 1[1] -> 2[2] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243412:243412 [1] NCCL INFO Channel 17/0 : 1[1] -> 2[2] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243412:243412 [1] NCCL INFO Channel 18/0 : 1[1] -> 2[2] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243412:243412 [1] NCCL INFO Channel 19/0 : 1[1] -> 2[2] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243412:243412 [1] NCCL INFO Channel 20/0 : 1[1] -> 2[2] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243412:243412 [1] NCCL INFO Channel 21/0 : 1[1] -> 2[2] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243412:243412 [1] NCCL INFO Channel 22/0 : 1[1] -> 2[2] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243412:243412 [1] NCCL INFO Channel 23/0 : 1[1] -> 2[2] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243418:243418 [7] NCCL INFO Channel 00/0 : 7[7] -> 0[0] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243418:243418 [7] NCCL INFO Channel 01/0 : 7[7] -> 0[0] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243418:243418 [7] NCCL INFO Channel 02/0 : 7[7] -> 0[0] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243418:243418 [7] NCCL INFO Channel 03/0 : 7[7] -> 0[0] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243418:243418 [7] NCCL INFO Channel 04/0 : 7[7] -> 0[0] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243418:243418 [7] NCCL INFO Channel 05/0 : 7[7] -> 0[0] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243418:243418 [7] NCCL INFO Channel 06/0 : 7[7] -> 0[0] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243418:243418 [7] NCCL INFO Channel 07/0 : 7[7] -> 0[0] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243418:243418 [7] NCCL INFO Channel 08/0 : 7[7] -> 0[0] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243418:243418 [7] NCCL INFO Channel 09/0 : 7[7] -> 0[0] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243418:243418 [7] NCCL INFO Channel 10/0 : 7[7] -> 0[0] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243418:243418 [7] NCCL INFO Channel 11/0 : 7[7] -> 0[0] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243418:243418 [7] NCCL INFO Channel 12/0 : 7[7] -> 0[0] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243418:243418 [7] NCCL INFO Channel 13/0 : 7[7] -> 0[0] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243418:243418 [7] NCCL INFO Channel 14/0 : 7[7] -> 0[0] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243418:243418 [7] NCCL INFO Channel 15/0 : 7[7] -> 0[0] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243418:243418 [7] NCCL INFO Channel 16/0 : 7[7] -> 0[0] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243418:243418 [7] NCCL INFO Channel 17/0 : 7[7] -> 0[0] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243418:243418 [7] NCCL INFO Channel 18/0 : 7[7] -> 0[0] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243418:243418 [7] NCCL INFO Channel 19/0 : 7[7] -> 0[0] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243418:243418 [7] NCCL INFO Channel 20/0 : 7[7] -> 0[0] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243418:243418 [7] NCCL INFO Channel 21/0 : 7[7] -> 0[0] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243418:243418 [7] NCCL INFO Channel 22/0 : 7[7] -> 0[0] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243418:243418 [7] NCCL INFO Channel 23/0 : 7[7] -> 0[0] via P2P/CUMEM/read +[INFO|2026-04-29 17:05:58] llamafactory.data.loader:144 >> Sampled 10000 examples from dataset allenai/tulu-3-sft-personas-code. +[INFO|2026-04-29 17:05:59] llamafactory.data.loader:144 >> Loading dataset KKHYA/evol_codealpaca_converted... +Repo card metadata block was not found. Setting CardData to empty. +[INFO|2026-04-29 17:06:00] llamafactory.data.loader:144 >> Sampled 10000 examples from dataset KKHYA/evol_codealpaca_converted. +[INFO|2026-04-29 17:06:00] llamafactory.data.loader:144 >> Loading dataset KKHYA/codefeedback_filtered_instructions_converted... +Repo card metadata block was not found. Setting CardData to empty. +[INFO|2026-04-29 17:06:01] llamafactory.data.loader:144 >> Sampled 10000 examples from dataset KKHYA/codefeedback_filtered_instructions_converted. +ywang29-p4d-debug-1-worker-0:243411:243411 [0] NCCL INFO Channel 00/0 : 0[0] -> 1[1] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243411:243411 [0] NCCL INFO Channel 01/0 : 0[0] -> 1[1] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243411:243411 [0] NCCL INFO Channel 02/0 : 0[0] -> 1[1] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243411:243411 [0] NCCL INFO Channel 03/0 : 0[0] -> 1[1] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243411:243411 [0] NCCL INFO Channel 04/0 : 0[0] -> 1[1] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243411:243411 [0] NCCL INFO Channel 05/0 : 0[0] -> 1[1] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243411:243411 [0] NCCL INFO Channel 06/0 : 0[0] -> 1[1] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243411:243411 [0] NCCL INFO Channel 07/0 : 0[0] -> 1[1] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243411:243411 [0] NCCL INFO Channel 08/0 : 0[0] -> 1[1] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243411:243411 [0] NCCL INFO Channel 09/0 : 0[0] -> 1[1] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243411:243411 [0] NCCL INFO Channel 10/0 : 0[0] -> 1[1] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243411:243411 [0] NCCL INFO Channel 11/0 : 0[0] -> 1[1] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243411:243411 [0] NCCL INFO Channel 12/0 : 0[0] -> 1[1] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243411:243411 [0] NCCL INFO Channel 13/0 : 0[0] -> 1[1] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243411:243411 [0] NCCL INFO Channel 14/0 : 0[0] -> 1[1] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243411:243411 [0] NCCL INFO Channel 15/0 : 0[0] -> 1[1] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243411:243411 [0] NCCL INFO Channel 16/0 : 0[0] -> 1[1] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243411:243411 [0] NCCL INFO Channel 17/0 : 0[0] -> 1[1] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243411:243411 [0] NCCL INFO Channel 18/0 : 0[0] -> 1[1] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243411:243411 [0] NCCL INFO Channel 19/0 : 0[0] -> 1[1] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243411:243411 [0] NCCL INFO Channel 20/0 : 0[0] -> 1[1] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243411:243411 [0] NCCL INFO Channel 21/0 : 0[0] -> 1[1] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243411:243411 [0] NCCL INFO Channel 22/0 : 0[0] -> 1[1] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243411:243411 [0] NCCL INFO Channel 23/0 : 0[0] -> 1[1] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243411:243411 [0] NCCL INFO Connected all rings, use ring PXN 0 GDR 1 +ywang29-p4d-debug-1-worker-0:243413:243413 [2] NCCL INFO Connected all rings, use ring PXN 0 GDR 1 +ywang29-p4d-debug-1-worker-0:243412:243412 [1] NCCL INFO Connected all rings, use ring PXN 0 GDR 1 +ywang29-p4d-debug-1-worker-0:243418:243418 [7] NCCL INFO Connected all rings, use ring PXN 0 GDR 1 +ywang29-p4d-debug-1-worker-0:243417:243417 [6] NCCL INFO Connected all rings, use ring PXN 0 GDR 1 +ywang29-p4d-debug-1-worker-0:243414:243414 [3] NCCL INFO Connected all rings, use ring PXN 0 GDR 1 +ywang29-p4d-debug-1-worker-0:243415:243415 [4] NCCL INFO Connected all rings, use ring PXN 0 GDR 1 +ywang29-p4d-debug-1-worker-0:243416:243416 [5] NCCL INFO Connected all rings, use ring PXN 0 GDR 1 +training example: +input_ids: +[151644, 872, 198, 7985, 264, 10135, 729, 311, 11047, 279, 2790, 1372, 315, 8845, 16548, 553, 1674, 3767, 300, 4308, 304, 264, 2661, 3200, 504, 264, 1140, 315, 2432, 3059, 13, 8886, 2432, 1102, 374, 15251, 438, 264, 10997, 448, 6894, 330, 5117, 26532, 497, 330, 13757, 26532, 497, 330, 5117, 96244, 497, 323, 330, 13757, 96244, 3263, 1674, 3767, 300, 4308, 1410, 387, 2987, 279, 2114, 476, 3123, 2083, 304, 894, 2432, 13, 576, 1946, 374, 264, 1140, 315, 1741, 2432, 1102, 57514, 11, 323, 279, 2550, 1265, 387, 458, 7546, 14064, 279, 2790, 1372, 315, 8845, 16548, 553, 1674, 3767, 300, 4308, 382, 2505, 510, 12, 1565, 6347, 13576, 44622, 362, 1140, 315, 57514, 11, 1380, 1817, 10997, 5610, 510, 220, 481, 330, 5117, 26532, 1, 320, 917, 1648, 576, 829, 315, 279, 2114, 2083, 624, 220, 481, 330, 13757, 26532, 1, 320, 917, 1648, 576, 829, 315, 279, 3123, 2083, 624, 220, 481, 330, 5117, 96244, 1, 320, 396, 1648, 576, 1372, 315, 8845, 16548, 553, 279, 2114, 2083, 624, 220, 481, 330, 13757, 96244, 1, 320, 396, 1648, 576, 1372, 315, 8845, 16548, 553, 279, 3123, 2083, 382, 5097, 510, 12, 1527, 7546, 14064, 279, 2790, 1372, 315, 8845, 16548, 553, 1674, 3767, 300, 4308, 382, 13314, 510, 73594, 12669, 198, 6347, 13576, 284, 2278, 262, 5212, 5117, 26532, 788, 330, 2101, 3767, 300, 4308, 497, 330, 13757, 26532, 788, 330, 14597, 32, 497, 330, 5117, 96244, 788, 220, 17, 11, 330, 13757, 96244, 788, 220, 16, 1583, 262, 5212, 5117, 26532, 788, 330, 14597, 33, 497, 330, 13757, 26532, 788, 330, 2101, 3767, 300, 4308, 497, 330, 5117, 96244, 788, 220, 18, 11, 330, 13757, 96244, 788, 220, 17, 1583, 262, 5212, 5117, 26532, 788, 330, 2101, 3767, 300, 4308, 497, 330, 13757, 26532, 788, 330, 14597, 34, 497, 330, 5117, 96244, 788, 220, 16, 11, 330, 13757, 96244, 788, 220, 16, 1583, 262, 5212, 5117, 26532, 788, 330, 14597, 35, 497, 330, 13757, 26532, 788, 330, 2101, 3767, 300, 4308, 497, 330, 5117, 96244, 788, 220, 15, 11, 330, 13757, 96244, 788, 220, 18, 532, 921, 2, 31021, 9258, 25, 220, 23, 198, 73594, 151645, 198, 151644, 77091, 198, 750, 11047, 8418, 3767, 300, 4308, 96244, 25401, 13576, 982, 262, 2790, 96244, 284, 220, 15, 198, 262, 369, 2432, 304, 2432, 13576, 510, 286, 421, 2432, 1183, 5117, 26532, 1341, 621, 330, 2101, 3767, 300, 4308, 4660, 310, 2790, 96244, 1421, 2432, 1183, 5117, 96244, 7026, 286, 4409, 2432, 1183, 13757, 26532, 1341, 621, 330, 2101, 3767, 300, 4308, 4660, 310, 2790, 96244, 1421, 2432, 1183, 13757, 96244, 7026, 262, 470, 2790, 96244, 151645, 198] +inputs: +<|im_start|>user +Write a python function to calculate the total number of goals scored by Alanyaspor in a given season from a list of match results. Each match result is represented as a dictionary with keys "home_team", "away_team", "home_goals", and "away_goals". Alanyaspor could be either the home or away team in any match. The input is a list of such match result dictionaries, and the output should be an integer representing the total number of goals scored by Alanyaspor. + +Input: +- `match_results`: A list of dictionaries, where each dictionary contains: + - "home_team" (string): The name of the home team. + - "away_team" (string): The name of the away team. + - "home_goals" (int): The number of goals scored by the home team. + - "away_goals" (int): The number of goals scored by the away team. + +Output: +- An integer representing the total number of goals scored by Alanyaspor. + +Example: +```python +match_results = [ + {"home_team": "Alanyaspor", "away_team": "TeamA", "home_goals": 2, "away_goals": 1}, + {"home_team": "TeamB", "away_team": "Alanyaspor", "home_goals": 3, "away_goals": 2}, + {"home_team": "Alanyaspor", "away_team": "TeamC", "home_goals": 1, "away_goals": 1}, + {"home_team": "TeamD", "away_team": "Alanyaspor", "home_goals": 0, "away_goals": 3} +] +# Expected Output: 8 +```<|im_end|> +<|im_start|>assistant +def calculate_alanyaspor_goals(match_results): + total_goals = 0 + for match in match_results: + if match["home_team"] == "Alanyaspor": + total_goals += match["home_goals"] + elif match["away_team"] == "Alanyaspor": + total_goals += match["away_goals"] + return total_goals<|im_end|> + +label_ids: +[-100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, 750, 11047, 8418, 3767, 300, 4308, 96244, 25401, 13576, 982, 262, 2790, 96244, 284, 220, 15, 198, 262, 369, 2432, 304, 2432, 13576, 510, 286, 421, 2432, 1183, 5117, 26532, 1341, 621, 330, 2101, 3767, 300, 4308, 4660, 310, 2790, 96244, 1421, 2432, 1183, 5117, 96244, 7026, 286, 4409, 2432, 1183, 13757, 26532, 1341, 621, 330, 2101, 3767, 300, 4308, 4660, 310, 2790, 96244, 1421, 2432, 1183, 13757, 96244, 7026, 262, 470, 2790, 96244, 151645, 198] +labels: +def calculate_alanyaspor_goals(match_results): + total_goals = 0 + for match in match_results: + if match["home_team"] == "Alanyaspor": + total_goals += match["home_goals"] + elif match["away_team"] == "Alanyaspor": + total_goals += match["away_goals"] + return total_goals<|im_end|> + +Repo card metadata block was not found. Setting CardData to empty. +Repo card metadata block was not found. Setting CardData to empty. +Repo card metadata block was not found. Setting CardData to empty. +Repo card metadata block was not found. Setting CardData to empty. +Repo card metadata block was not found. Setting CardData to empty. +Repo card metadata block was not found. Setting CardData to empty. +Repo card metadata block was not found. Setting CardData to empty. +Repo card metadata block was not found. Setting CardData to empty. +Repo card metadata block was not found. Setting CardData to empty. +Repo card metadata block was not found. Setting CardData to empty. +Repo card metadata block was not found. Setting CardData to empty. +Repo card metadata block was not found. Setting CardData to empty. +Repo card metadata block was not found. Setting CardData to empty. +Repo card metadata block was not found. Setting CardData to empty. +[INFO|configuration_utils.py:670] 2026-04-29 17:06:05,518 >> loading configuration file config.json from cache at /root/.cache/huggingface/hub/models--Qwen--Qwen3-1.7B/snapshots/70d244cc86ccca08cf5af4e1e306ecf908b1ad5e/config.json +[INFO|configuration_utils.py:742] 2026-04-29 17:06:05,518 >> Model config Qwen3Config { + "architectures": [ + "Qwen3ForCausalLM" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "bos_token_id": 151643, + "dtype": "bfloat16", + "eos_token_id": 151645, + "head_dim": 128, + "hidden_act": "silu", + "hidden_size": 2048, + "initializer_range": 0.02, + "intermediate_size": 6144, + "layer_types": [ + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention" + ], + "masked_layers": null, + "max_position_embeddings": 40960, + "max_window_layers": 28, + "model_type": "qwen3", + "num_attention_heads": 16, + "num_hidden_layers": 28, + "num_key_value_heads": 8, + "pad_token_id": null, + "rms_norm_eps": 1e-06, + "rope_parameters": { + "rope_theta": 1000000, + "rope_type": "default" + }, + "sliding_window": null, + "sparsity_attn": null, + "sparsity_mlp": null, + "subnet_mode": null, + "subnet_type": null, + "threshold_attn": null, + "threshold_mlp": null, + "tie_word_embeddings": true, + "transformers_version": "5.2.0", + "use_cache": true, + "use_sliding_window": false, + "vocab_size": 151936 +} + +[INFO|2026-04-29 17:06:05] llamafactory.model.model_utils.kv_cache:144 >> KV cache is disabled during training. +[INFO|modeling_utils.py:710] 2026-04-29 17:06:06,257 >> loading weights file model.safetensors from cache at /root/.cache/huggingface/hub/models--Qwen--Qwen3-1.7B/snapshots/70d244cc86ccca08cf5af4e1e306ecf908b1ad5e/model.safetensors.index.json +[INFO|modeling_utils.py:779] 2026-04-29 17:06:06,257 >> Will use dtype=torch.bfloat16 as defined in model's config object +[INFO|configuration_utils.py:1014] 2026-04-29 17:06:06,258 >> Generate config GenerationConfig { + "bos_token_id": 151643, + "eos_token_id": 151645, + "output_attentions": false, + "output_hidden_states": false, + "use_cache": false +} + + Loading weights: 0%| | 0/311 [00:00> The tied weights mapping and config for this model specifies to tie model.embed_tokens.weight to lm_head.weight, but both are present in the checkpoints, so we will NOT tie them. You should update the config with `tie_word_embeddings=False` to silence this warning + Loading weights: 1%| | 2/311 [00:00<00:24, 12.67it/s, Materializing param=lm_head.weight] Loading weights: 1%| | 2/311 [00:00<00:24, 12.67it/s, Materializing param=model.embed_tokens.weight] Loading weights: 1%| | 2/311 [00:00<00:24, 12.67it/s, Materializing param=model.embed_tokens.weight] Loading weights: 1%| | 2/311 [00:00<00:42, 7.33it/s, Materializing param=lm_head.weight] Loading weights: 1%| | 2/311 [00:00<00:42, 7.33it/s, Materializing param=model.embed_tokens.weight] Loading weights: 1%| | 2/311 [00:00<00:42, 7.33it/s, Materializing param=model.embed_tokens.weight] Loading weights: 1%| | 3/311 [00:00<00:42, 7.33it/s, Materializing param=model.layers.0.input_layernorm.weight] Loading weights: 1%| | 3/311 [00:00<00:42, 7.33it/s, Materializing param=model.layers.0.input_layernorm.weight] Loading weights: 1%|▏ | 4/311 [00:00<00:41, 7.33it/s, Materializing param=model.layers.0.mlp.down_proj.weight] Loading weights: 1%|▏ | 4/311 [00:00<00:41, 7.33it/s, Materializing param=model.layers.0.mlp.down_proj.weight] Loading weights: 2%|▏ | 5/311 [00:00<00:41, 7.33it/s, Materializing param=model.layers.0.mlp.gate_proj.weight] Loading weights: 2%|▏ | 5/311 [00:00<00:41, 7.33it/s, Materializing param=model.layers.0.mlp.gate_proj.weight] Loading weights: 1%| | 2/311 [00:00<00:43, 7.14it/s, Materializing param=lm_head.weight] Loading weights: 1%| | 2/311 [00:00<00:43, 7.14it/s, Materializing param=model.embed_tokens.weight] Loading weights: 1%| | 2/311 [00:00<00:43, 7.14it/s, Materializing param=model.embed_tokens.weight] Loading weights: 1%| | 3/311 [00:00<00:43, 7.14it/s, Materializing param=model.layers.0.input_layernorm.weight] Loading weights: 1%| | 3/311 [00:00<00:43, 7.14it/s, Materializing param=model.layers.0.input_layernorm.weight] Loading weights: 1%|▏ | 4/311 [00:00<00:43, 7.14it/s, Materializing param=model.layers.0.mlp.down_proj.weight] Loading weights: 1%|▏ | 4/311 [00:00<00:43, 7.14it/s, Materializing param=model.layers.0.mlp.down_proj.weight] Loading weights: 2%|▏ | 6/311 [00:00<00:41, 7.33it/s, Materializing param=model.layers.0.mlp.up_proj.weight] Loading weights: 2%|▏ | 6/311 [00:00<00:41, 7.33it/s, Materializing param=model.layers.0.mlp.up_proj.weight] Loading weights: 2%|▏ | 7/311 [00:00<00:41, 7.33it/s, Materializing param=model.layers.0.post_attention_layernorm.weight] Loading weights: 2%|▏ | 7/311 [00:00<00:41, 7.33it/s, Materializing param=model.layers.0.post_attention_layernorm.weight] Loading weights: 3%|▎ | 8/311 [00:00<00:41, 7.33it/s, Materializing param=model.layers.0.self_attn.k_norm.weight] Loading weights: 3%|▎ | 8/311 [00:00<00:41, 7.33it/s, Materializing param=model.layers.0.self_attn.k_norm.weight] Loading weights: 3%|▎ | 9/311 [00:00<00:41, 7.33it/s, Materializing param=model.layers.0.self_attn.k_proj.weight] Loading weights: 3%|▎ | 9/311 [00:00<00:41, 7.33it/s, Materializing param=model.layers.0.self_attn.k_proj.weight] Loading weights: 3%|▎ | 10/311 [00:00<00:41, 7.33it/s, Materializing param=model.layers.0.self_attn.o_proj.weight] Loading weights: 3%|▎ | 10/311 [00:00<00:41, 7.33it/s, Materializing param=model.layers.0.self_attn.o_proj.weight] Loading weights: 2%|▏ | 5/311 [00:00<00:42, 7.14it/s, Materializing param=model.layers.0.mlp.gate_proj.weight] Loading weights: 2%|▏ | 5/311 [00:00<00:42, 7.14it/s, Materializing param=model.layers.0.mlp.gate_proj.weight] Loading weights: 4%|▎ | 11/311 [00:00<00:40, 7.33it/s, Materializing param=model.layers.0.self_attn.q_norm.weight] Loading weights: 2%|▏ | 6/311 [00:00<00:42, 7.14it/s, Materializing param=model.layers.0.mlp.up_proj.weight] Loading weights: 4%|▎ | 11/311 [00:00<00:40, 7.33it/s, Materializing param=model.layers.0.self_attn.q_norm.weight] Loading weights: 2%|▏ | 6/311 [00:00<00:42, 7.14it/s, Materializing param=model.layers.0.mlp.up_proj.weight] Loading weights: 4%|▍ | 12/311 [00:00<00:40, 7.33it/s, Materializing param=model.layers.0.self_attn.q_proj.weight] Loading weights: 4%|▍ | 12/311 [00:00<00:40, 7.33it/s, Materializing param=model.layers.0.self_attn.q_proj.weight] Loading weights: 4%|▍ | 13/311 [00:00<00:40, 7.33it/s, Materializing param=model.layers.0.self_attn.v_proj.weight] Loading weights: 4%|▍ | 13/311 [00:00<00:40, 7.33it/s, Materializing param=model.layers.0.self_attn.v_proj.weight] Loading weights: 5%|▍ | 14/311 [00:00<00:40, 7.33it/s, Materializing param=model.layers.1.input_layernorm.weight] Loading weights: 5%|▍ | 14/311 [00:00<00:40, 7.33it/s, Materializing param=model.layers.1.input_layernorm.weight] Loading weights: 5%|▍ | 15/311 [00:00<00:40, 7.33it/s, Materializing param=model.layers.1.mlp.down_proj.weight] Loading weights: 5%|▍ | 15/311 [00:00<00:40, 7.33it/s, Materializing param=model.layers.1.mlp.down_proj.weight] Loading weights: 1%| | 3/311 [00:00<00:18, 16.73it/s, Materializing param=model.layers.0.input_layernorm.weight] Loading weights: 1%| | 3/311 [00:00<00:18, 16.73it/s, Materializing param=model.layers.0.input_layernorm.weight] Loading weights: 1%|▏ | 4/311 [00:00<00:22, 13.86it/s, Materializing param=model.layers.0.input_layernorm.weight] Loading weights: 1%|▏ | 4/311 [00:00<00:22, 13.86it/s, Materializing param=model.layers.0.mlp.down_proj.weight] Loading weights: 1%|▏ | 4/311 [00:00<00:22, 13.86it/s, Materializing param=model.layers.0.mlp.down_proj.weight] Loading weights: 2%|▏ | 7/311 [00:00<00:42, 7.14it/s, Materializing param=model.layers.0.post_attention_layernorm.weight] Loading weights: 2%|▏ | 7/311 [00:00<00:42, 7.14it/s, Materializing param=model.layers.0.post_attention_layernorm.weight] Loading weights: 3%|▎ | 8/311 [00:00<00:42, 7.14it/s, Materializing param=model.layers.0.self_attn.k_norm.weight] Loading weights: 3%|▎ | 8/311 [00:00<00:42, 7.14it/s, Materializing param=model.layers.0.self_attn.k_norm.weight] Loading weights: 3%|▎ | 9/311 [00:00<00:42, 7.14it/s, Materializing param=model.layers.0.self_attn.k_proj.weight] Loading weights: 3%|▎ | 9/311 [00:00<00:42, 7.14it/s, Materializing param=model.layers.0.self_attn.k_proj.weight] Loading weights: 3%|▎ | 10/311 [00:00<00:42, 7.14it/s, Materializing param=model.layers.0.self_attn.o_proj.weight] Loading weights: 3%|▎ | 10/311 [00:00<00:42, 7.14it/s, Materializing param=model.layers.0.self_attn.o_proj.weight] Loading weights: 5%|▌ | 16/311 [00:00<00:40, 7.33it/s, Materializing param=model.layers.1.mlp.gate_proj.weight] Loading weights: 5%|▌ | 16/311 [00:00<00:40, 7.33it/s, Materializing param=model.layers.1.mlp.gate_proj.weight] Loading weights: 4%|▎ | 11/311 [00:00<00:42, 7.14it/s, Materializing param=model.layers.0.self_attn.q_norm.weight] Loading weights: 4%|▎ | 11/311 [00:00<00:42, 7.14it/s, Materializing param=model.layers.0.self_attn.q_norm.weight] Loading weights: 4%|▍ | 12/311 [00:00<00:41, 7.14it/s, Materializing param=model.layers.0.self_attn.q_proj.weight] Loading weights: 4%|▍ | 12/311 [00:00<00:41, 7.14it/s, Materializing param=model.layers.0.self_attn.q_proj.weight] Loading weights: 4%|▍ | 13/311 [00:00<00:41, 7.14it/s, Materializing param=model.layers.0.self_attn.v_proj.weight] Loading weights: 4%|▍ | 13/311 [00:00<00:41, 7.14it/s, Materializing param=model.layers.0.self_attn.v_proj.weight] Loading weights: 2%|▏ | 5/311 [00:00<00:22, 13.86it/s, Materializing param=model.layers.0.mlp.gate_proj.weight] Loading weights: 2%|▏ | 5/311 [00:00<00:22, 13.86it/s, Materializing param=model.layers.0.mlp.gate_proj.weight] Loading weights: 5%|▍ | 14/311 [00:00<00:41, 7.14it/s, Materializing param=model.layers.1.input_layernorm.weight] Loading weights: 5%|▍ | 14/311 [00:00<00:41, 7.14it/s, Materializing param=model.layers.1.input_layernorm.weight] Loading weights: 5%|▍ | 15/311 [00:00<00:41, 7.14it/s, Materializing param=model.layers.1.mlp.down_proj.weight] Loading weights: 5%|▍ | 15/311 [00:00<00:41, 7.14it/s, Materializing param=model.layers.1.mlp.down_proj.weight] Loading weights: 1%| | 2/311 [00:00<00:40, 7.59it/s, Materializing param=lm_head.weight] Loading weights: 1%| | 2/311 [00:00<00:40, 7.59it/s, Materializing param=model.embed_tokens.weight] Loading weights: 5%|▌ | 17/311 [00:00<00:40, 7.33it/s, Materializing param=model.layers.1.mlp.up_proj.weight] Loading weights: 1%| | 2/311 [00:00<00:40, 7.59it/s, Materializing param=model.embed_tokens.weight] Loading weights: 5%|▌ | 17/311 [00:00<00:40, 7.33it/s, Materializing param=model.layers.1.mlp.up_proj.weight] Loading weights: 1%| | 3/311 [00:00<00:40, 7.59it/s, Materializing param=model.layers.0.input_layernorm.weight] Loading weights: 1%| | 3/311 [00:00<00:40, 7.59it/s, Materializing param=model.layers.0.input_layernorm.weight] Loading weights: 1%|▏ | 4/311 [00:00<00:40, 7.59it/s, Materializing param=model.layers.0.mlp.down_proj.weight] Loading weights: 1%|▏ | 4/311 [00:00<00:40, 7.59it/s, Materializing param=model.layers.0.mlp.down_proj.weight] Loading weights: 2%|▏ | 6/311 [00:00<00:22, 13.86it/s, Materializing param=model.layers.0.mlp.up_proj.weight] Loading weights: 2%|▏ | 6/311 [00:00<00:22, 13.86it/s, Materializing param=model.layers.0.mlp.up_proj.weight] Loading weights: 6%|▌ | 18/311 [00:00<00:39, 7.33it/s, Materializing param=model.layers.1.post_attention_layernorm.weight] Loading weights: 6%|▌ | 18/311 [00:00<00:39, 7.33it/s, Materializing param=model.layers.1.post_attention_layernorm.weight] Loading weights: 2%|▏ | 5/311 [00:00<00:40, 7.59it/s, Materializing param=model.layers.0.mlp.gate_proj.weight] Loading weights: 2%|▏ | 5/311 [00:00<00:40, 7.59it/s, Materializing param=model.layers.0.mlp.gate_proj.weight] Loading weights: 6%|▌ | 19/311 [00:00<00:39, 7.33it/s, Materializing param=model.layers.1.self_attn.k_norm.weight] Loading weights: 6%|▌ | 19/311 [00:00<00:39, 7.33it/s, Materializing param=model.layers.1.self_attn.k_norm.weight] Loading weights: 6%|▋ | 20/311 [00:00<00:39, 7.33it/s, Materializing param=model.layers.1.self_attn.k_proj.weight] Loading weights: 6%|▋ | 20/311 [00:00<00:39, 7.33it/s, Materializing param=model.layers.1.self_attn.k_proj.weight] Loading weights: 1%| | 2/311 [00:00<00:49, 6.28it/s, Materializing param=lm_head.weight] Loading weights: 1%| | 2/311 [00:00<00:49, 6.28it/s, Materializing param=model.embed_tokens.weight] Loading weights: 1%| | 2/311 [00:00<00:49, 6.28it/s, Materializing param=model.embed_tokens.weight] Loading weights: 1%| | 3/311 [00:00<00:49, 6.28it/s, Materializing param=model.layers.0.input_layernorm.weight] Loading weights: 1%| | 3/311 [00:00<00:49, 6.28it/s, Materializing param=model.layers.0.input_layernorm.weight] Loading weights: 7%|▋ | 21/311 [00:00<00:39, 7.33it/s, Materializing param=model.layers.1.self_attn.o_proj.weight] Loading weights: 7%|▋ | 21/311 [00:00<00:39, 7.33it/s, Materializing param=model.layers.1.self_attn.o_proj.weight] Loading weights: 2%|▏ | 7/311 [00:00<00:21, 13.86it/s, Materializing param=model.layers.0.post_attention_layernorm.weight] Loading weights: 2%|▏ | 7/311 [00:00<00:21, 13.86it/s, Materializing param=model.layers.0.post_attention_layernorm.weight] Loading weights: 3%|▎ | 8/311 [00:00<00:21, 13.86it/s, Materializing param=model.layers.0.self_attn.k_norm.weight] Loading weights: 3%|▎ | 8/311 [00:00<00:21, 13.86it/s, Materializing param=model.layers.0.self_attn.k_norm.weight] Loading weights: 3%|▎ | 9/311 [00:00<00:21, 13.86it/s, Materializing param=model.layers.0.self_attn.k_proj.weight] Loading weights: 3%|▎ | 9/311 [00:00<00:21, 13.86it/s, Materializing param=model.layers.0.self_attn.k_proj.weight] Loading weights: 3%|▎ | 10/311 [00:00<00:21, 13.86it/s, Materializing param=model.layers.0.self_attn.o_proj.weight] Loading weights: 3%|▎ | 10/311 [00:00<00:21, 13.86it/s, Materializing param=model.layers.0.self_attn.o_proj.weight] Loading weights: 7%|▋ | 22/311 [00:00<00:39, 7.33it/s, Materializing param=model.layers.1.self_attn.q_norm.weight] Loading weights: 7%|▋ | 22/311 [00:00<00:39, 7.33it/s, Materializing param=model.layers.1.self_attn.q_norm.weight] Loading weights: 7%|▋ | 23/311 [00:00<00:39, 7.33it/s, Materializing param=model.layers.1.self_attn.q_proj.weight] Loading weights: 7%|▋ | 23/311 [00:00<00:39, 7.33it/s, Materializing param=model.layers.1.self_attn.q_proj.weight] Loading weights: 8%|▊ | 24/311 [00:00<00:39, 7.33it/s, Materializing param=model.layers.1.self_attn.v_proj.weight] Loading weights: 8%|▊ | 24/311 [00:00<00:39, 7.33it/s, Materializing param=model.layers.1.self_attn.v_proj.weight] Loading weights: 5%|▌ | 16/311 [00:00<00:41, 7.14it/s, Materializing param=model.layers.1.mlp.gate_proj.weight] Loading weights: 5%|▌ | 16/311 [00:00<00:41, 7.14it/s, Materializing param=model.layers.1.mlp.gate_proj.weight] Loading weights: 5%|▌ | 17/311 [00:00<00:41, 7.14it/s, Materializing param=model.layers.1.mlp.up_proj.weight] Loading weights: 5%|▌ | 17/311 [00:00<00:41, 7.14it/s, Materializing param=model.layers.1.mlp.up_proj.weight] Loading weights: 8%|▊ | 25/311 [00:00<00:39, 7.33it/s, Materializing param=model.layers.2.input_layernorm.weight] Loading weights: 8%|▊ | 25/311 [00:00<00:39, 7.33it/s, Materializing param=model.layers.2.input_layernorm.weight] Loading weights: 8%|▊ | 26/311 [00:00<00:38, 7.33it/s, Materializing param=model.layers.2.mlp.down_proj.weight] Loading weights: 8%|▊ | 26/311 [00:00<00:38, 7.33it/s, Materializing param=model.layers.2.mlp.down_proj.weight] Loading weights: 4%|▎ | 11/311 [00:00<00:21, 13.86it/s, Materializing param=model.layers.0.self_attn.q_norm.weight] Loading weights: 4%|▎ | 11/311 [00:00<00:21, 13.86it/s, Materializing param=model.layers.0.self_attn.q_norm.weight] Loading weights: 4%|▍ | 12/311 [00:00<00:21, 13.86it/s, Materializing param=model.layers.0.self_attn.q_proj.weight] Loading weights: 1%|▏ | 4/311 [00:00<00:48, 6.28it/s, Materializing param=model.layers.0.mlp.down_proj.weight] Loading weights: 4%|▍ | 12/311 [00:00<00:21, 13.86it/s, Materializing param=model.layers.0.self_attn.q_proj.weight] Loading weights: 1%|▏ | 4/311 [00:00<00:48, 6.28it/s, Materializing param=model.layers.0.mlp.down_proj.weight] Loading weights: 4%|▍ | 13/311 [00:00<00:21, 13.86it/s, Materializing param=model.layers.0.self_attn.v_proj.weight] Loading weights: 2%|▏ | 5/311 [00:00<00:48, 6.28it/s, Materializing param=model.layers.0.mlp.gate_proj.weight] Loading weights: 4%|▍ | 13/311 [00:00<00:21, 13.86it/s, Materializing param=model.layers.0.self_attn.v_proj.weight] Loading weights: 2%|▏ | 5/311 [00:00<00:48, 6.28it/s, Materializing param=model.layers.0.mlp.gate_proj.weight] Loading weights: 5%|▍ | 14/311 [00:00<00:21, 13.86it/s, Materializing param=model.layers.1.input_layernorm.weight] Loading weights: 5%|▍ | 14/311 [00:00<00:21, 13.86it/s, Materializing param=model.layers.1.input_layernorm.weight] Loading weights: 5%|▍ | 15/311 [00:00<00:21, 13.86it/s, Materializing param=model.layers.1.mlp.down_proj.weight] Loading weights: 5%|▍ | 15/311 [00:00<00:21, 13.86it/s, Materializing param=model.layers.1.mlp.down_proj.weight] Loading weights: 2%|▏ | 6/311 [00:00<00:40, 7.59it/s, Materializing param=model.layers.0.mlp.up_proj.weight] Loading weights: 2%|▏ | 6/311 [00:00<00:40, 7.59it/s, Materializing param=model.layers.0.mlp.up_proj.weight] Loading weights: 9%|▊ | 27/311 [00:00<00:38, 7.33it/s, Materializing param=model.layers.2.mlp.gate_proj.weight] Loading weights: 9%|▊ | 27/311 [00:00<00:38, 7.33it/s, Materializing param=model.layers.2.mlp.gate_proj.weight] Loading weights: 2%|▏ | 7/311 [00:00<00:40, 7.59it/s, Materializing param=model.layers.0.post_attention_layernorm.weight] Loading weights: 2%|▏ | 7/311 [00:00<00:40, 7.59it/s, Materializing param=model.layers.0.post_attention_layernorm.weight] Loading weights: 6%|▌ | 18/311 [00:00<00:41, 7.14it/s, Materializing param=model.layers.1.post_attention_layernorm.weight] Loading weights: 6%|▌ | 18/311 [00:00<00:41, 7.14it/s, Materializing param=model.layers.1.post_attention_layernorm.weight] Loading weights: 3%|▎ | 8/311 [00:00<00:39, 7.59it/s, Materializing param=model.layers.0.self_attn.k_norm.weight] Loading weights: 6%|▌ | 19/311 [00:00<00:40, 7.14it/s, Materializing param=model.layers.1.self_attn.k_norm.weight] Loading weights: 3%|▎ | 8/311 [00:00<00:39, 7.59it/s, Materializing param=model.layers.0.self_attn.k_norm.weight] Loading weights: 6%|▌ | 19/311 [00:00<00:40, 7.14it/s, Materializing param=model.layers.1.self_attn.k_norm.weight] Loading weights: 6%|▋ | 20/311 [00:00<00:40, 7.14it/s, Materializing param=model.layers.1.self_attn.k_proj.weight] Loading weights: 6%|▋ | 20/311 [00:00<00:40, 7.14it/s, Materializing param=model.layers.1.self_attn.k_proj.weight] Loading weights: 3%|▎ | 9/311 [00:00<00:39, 7.59it/s, Materializing param=model.layers.0.self_attn.k_proj.weight] Loading weights: 3%|▎ | 9/311 [00:00<00:39, 7.59it/s, Materializing param=model.layers.0.self_attn.k_proj.weight] Loading weights: 5%|▌ | 16/311 [00:00<00:21, 13.86it/s, Materializing param=model.layers.1.mlp.gate_proj.weight] Loading weights: 7%|▋ | 21/311 [00:00<00:40, 7.14it/s, Materializing param=model.layers.1.self_attn.o_proj.weight] Loading weights: 5%|▌ | 16/311 [00:00<00:21, 13.86it/s, Materializing param=model.layers.1.mlp.gate_proj.weight] Loading weights: 7%|▋ | 21/311 [00:00<00:40, 7.14it/s, Materializing param=model.layers.1.self_attn.o_proj.weight] Loading weights: 3%|▎ | 10/311 [00:00<00:39, 7.59it/s, Materializing param=model.layers.0.self_attn.o_proj.weight] Loading weights: 3%|▎ | 10/311 [00:00<00:39, 7.59it/s, Materializing param=model.layers.0.self_attn.o_proj.weight] Loading weights: 2%|▏ | 6/311 [00:00<00:48, 6.28it/s, Materializing param=model.layers.0.mlp.up_proj.weight] Loading weights: 2%|▏ | 6/311 [00:00<00:48, 6.28it/s, Materializing param=model.layers.0.mlp.up_proj.weight] Loading weights: 7%|▋ | 22/311 [00:00<00:40, 7.14it/s, Materializing param=model.layers.1.self_attn.q_norm.weight] Loading weights: 7%|▋ | 22/311 [00:00<00:40, 7.14it/s, Materializing param=model.layers.1.self_attn.q_norm.weight] Loading weights: 7%|▋ | 23/311 [00:00<00:40, 7.14it/s, Materializing param=model.layers.1.self_attn.q_proj.weight] Loading weights: 7%|▋ | 23/311 [00:00<00:40, 7.14it/s, Materializing param=model.layers.1.self_attn.q_proj.weight] Loading weights: 8%|▊ | 24/311 [00:00<00:40, 7.14it/s, Materializing param=model.layers.1.self_attn.v_proj.weight] Loading weights: 8%|▊ | 24/311 [00:00<00:40, 7.14it/s, Materializing param=model.layers.1.self_attn.v_proj.weight] Loading weights: 4%|▎ | 11/311 [00:00<00:39, 7.59it/s, Materializing param=model.layers.0.self_attn.q_norm.weight] Loading weights: 5%|▌ | 17/311 [00:00<00:21, 13.86it/s, Materializing param=model.layers.1.mlp.up_proj.weight] Loading weights: 4%|▎ | 11/311 [00:00<00:39, 7.59it/s, Materializing param=model.layers.0.self_attn.q_norm.weight] Loading weights: 5%|▌ | 17/311 [00:00<00:21, 13.86it/s, Materializing param=model.layers.1.mlp.up_proj.weight] Loading weights: 4%|▍ | 12/311 [00:00<00:39, 7.59it/s, Materializing param=model.layers.0.self_attn.q_proj.weight] Loading weights: 4%|▍ | 12/311 [00:00<00:39, 7.59it/s, Materializing param=model.layers.0.self_attn.q_proj.weight] Loading weights: 4%|▍ | 13/311 [00:00<00:39, 7.59it/s, Materializing param=model.layers.0.self_attn.v_proj.weight] Loading weights: 4%|▍ | 13/311 [00:00<00:39, 7.59it/s, Materializing param=model.layers.0.self_attn.v_proj.weight] Loading weights: 9%|▉ | 28/311 [00:00<00:38, 7.33it/s, Materializing param=model.layers.2.mlp.up_proj.weight] Loading weights: 9%|▉ | 28/311 [00:00<00:38, 7.33it/s, Materializing param=model.layers.2.mlp.up_proj.weight] Loading weights: 8%|▊ | 25/311 [00:00<00:40, 7.14it/s, Materializing param=model.layers.2.input_layernorm.weight] Loading weights: 5%|▍ | 14/311 [00:00<00:39, 7.59it/s, Materializing param=model.layers.1.input_layernorm.weight] Loading weights: 8%|▊ | 25/311 [00:00<00:40, 7.14it/s, Materializing param=model.layers.2.input_layernorm.weight] Loading weights: 5%|▍ | 14/311 [00:00<00:39, 7.59it/s, Materializing param=model.layers.1.input_layernorm.weight] Loading weights: 8%|▊ | 26/311 [00:00<00:39, 7.14it/s, Materializing param=model.layers.2.mlp.down_proj.weight] Loading weights: 8%|▊ | 26/311 [00:00<00:39, 7.14it/s, Materializing param=model.layers.2.mlp.down_proj.weight] Loading weights: 5%|▍ | 15/311 [00:00<00:38, 7.59it/s, Materializing param=model.layers.1.mlp.down_proj.weight] Loading weights: 5%|▍ | 15/311 [00:00<00:38, 7.59it/s, Materializing param=model.layers.1.mlp.down_proj.weight] Loading weights: 2%|▏ | 7/311 [00:00<00:48, 6.28it/s, Materializing param=model.layers.0.post_attention_layernorm.weight] Loading weights: 2%|▏ | 7/311 [00:00<00:48, 6.28it/s, Materializing param=model.layers.0.post_attention_layernorm.weight] Loading weights: 3%|▎ | 8/311 [00:00<00:48, 6.28it/s, Materializing param=model.layers.0.self_attn.k_norm.weight] Loading weights: 3%|▎ | 8/311 [00:00<00:48, 6.28it/s, Materializing param=model.layers.0.self_attn.k_norm.weight] Loading weights: 3%|▎ | 9/311 [00:00<00:48, 6.28it/s, Materializing param=model.layers.0.self_attn.k_proj.weight] Loading weights: 3%|▎ | 9/311 [00:00<00:48, 6.28it/s, Materializing param=model.layers.0.self_attn.k_proj.weight] Loading weights: 3%|▎ | 10/311 [00:00<00:47, 6.28it/s, Materializing param=model.layers.0.self_attn.o_proj.weight] Loading weights: 3%|▎ | 10/311 [00:00<00:47, 6.28it/s, Materializing param=model.layers.0.self_attn.o_proj.weight] Loading weights: 5%|▌ | 16/311 [00:00<00:38, 7.59it/s, Materializing param=model.layers.1.mlp.gate_proj.weight] Loading weights: 5%|▌ | 16/311 [00:00<00:38, 7.59it/s, Materializing param=model.layers.1.mlp.gate_proj.weight] Loading weights: 6%|▌ | 18/311 [00:00<00:21, 13.86it/s, Materializing param=model.layers.1.post_attention_layernorm.weight] Loading weights: 6%|▌ | 18/311 [00:00<00:21, 13.86it/s, Materializing param=model.layers.1.post_attention_layernorm.weight] Loading weights: 6%|▌ | 19/311 [00:00<00:21, 13.86it/s, Materializing param=model.layers.1.self_attn.k_norm.weight] Loading weights: 6%|▌ | 19/311 [00:00<00:21, 13.86it/s, Materializing param=model.layers.1.self_attn.k_norm.weight] Loading weights: 6%|▋ | 20/311 [00:00<00:20, 13.86it/s, Materializing param=model.layers.1.self_attn.k_proj.weight] Loading weights: 6%|▋ | 20/311 [00:00<00:20, 13.86it/s, Materializing param=model.layers.1.self_attn.k_proj.weight] Loading weights: 9%|▊ | 27/311 [00:00<00:39, 7.14it/s, Materializing param=model.layers.2.mlp.gate_proj.weight] Loading weights: 9%|▊ | 27/311 [00:00<00:39, 7.14it/s, Materializing param=model.layers.2.mlp.gate_proj.weight] Loading weights: 7%|▋ | 21/311 [00:00<00:20, 13.86it/s, Materializing param=model.layers.1.self_attn.o_proj.weight] Loading weights: 7%|▋ | 21/311 [00:00<00:20, 13.86it/s, Materializing param=model.layers.1.self_attn.o_proj.weight] Loading weights: 4%|▎ | 11/311 [00:00<00:47, 6.28it/s, Materializing param=model.layers.0.self_attn.q_norm.weight] Loading weights: 4%|▎ | 11/311 [00:00<00:47, 6.28it/s, Materializing param=model.layers.0.self_attn.q_norm.weight] Loading weights: 9%|▉ | 29/311 [00:00<00:38, 7.33it/s, Materializing param=model.layers.2.post_attention_layernorm.weight] Loading weights: 9%|▉ | 29/311 [00:00<00:38, 7.33it/s, Materializing param=model.layers.2.post_attention_layernorm.weight] Loading weights: 10%|▉ | 30/311 [00:00<00:38, 7.33it/s, Materializing param=model.layers.2.self_attn.k_norm.weight] Loading weights: 7%|▋ | 22/311 [00:00<00:20, 13.86it/s, Materializing param=model.layers.1.self_attn.q_norm.weight] Loading weights: 10%|▉ | 30/311 [00:00<00:38, 7.33it/s, Materializing param=model.layers.2.self_attn.k_norm.weight] Loading weights: 7%|▋ | 22/311 [00:00<00:20, 13.86it/s, Materializing param=model.layers.1.self_attn.q_norm.weight] Loading weights: 10%|▉ | 31/311 [00:00<00:38, 7.33it/s, Materializing param=model.layers.2.self_attn.k_proj.weight] Loading weights: 10%|▉ | 31/311 [00:00<00:38, 7.33it/s, Materializing param=model.layers.2.self_attn.k_proj.weight] Loading weights: 7%|▋ | 23/311 [00:00<00:20, 13.86it/s, Materializing param=model.layers.1.self_attn.q_proj.weight] Loading weights: 7%|▋ | 23/311 [00:00<00:20, 13.86it/s, Materializing param=model.layers.1.self_attn.q_proj.weight] Loading weights: 10%|█ | 32/311 [00:00<00:38, 7.33it/s, Materializing param=model.layers.2.self_attn.o_proj.weight] Loading weights: 10%|█ | 32/311 [00:00<00:38, 7.33it/s, Materializing param=model.layers.2.self_attn.o_proj.weight] Loading weights: 4%|▍ | 12/311 [00:00<00:47, 6.28it/s, Materializing param=model.layers.0.self_attn.q_proj.weight] Loading weights: 4%|▍ | 12/311 [00:00<00:47, 6.28it/s, Materializing param=model.layers.0.self_attn.q_proj.weight] Loading weights: 4%|▍ | 13/311 [00:00<00:47, 6.28it/s, Materializing param=model.layers.0.self_attn.v_proj.weight] Loading weights: 4%|▍ | 13/311 [00:00<00:47, 6.28it/s, Materializing param=model.layers.0.self_attn.v_proj.weight] Loading weights: 8%|▊ | 24/311 [00:00<00:20, 13.86it/s, Materializing param=model.layers.1.self_attn.v_proj.weight] Loading weights: 8%|▊ | 24/311 [00:00<00:20, 13.86it/s, Materializing param=model.layers.1.self_attn.v_proj.weight] Loading weights: 5%|▍ | 14/311 [00:00<00:47, 6.28it/s, Materializing param=model.layers.1.input_layernorm.weight] Loading weights: 11%|█ | 33/311 [00:00<00:37, 7.33it/s, Materializing param=model.layers.2.self_attn.q_norm.weight] Loading weights: 5%|▍ | 14/311 [00:00<00:47, 6.28it/s, Materializing param=model.layers.1.input_layernorm.weight] Loading weights: 11%|█ | 33/311 [00:00<00:37, 7.33it/s, Materializing param=model.layers.2.self_attn.q_norm.weight] Loading weights: 5%|▍ | 15/311 [00:00<00:47, 6.28it/s, Materializing param=model.layers.1.mlp.down_proj.weight] Loading weights: 5%|▍ | 15/311 [00:00<00:47, 6.28it/s, Materializing param=model.layers.1.mlp.down_proj.weight] Loading weights: 11%|█ | 34/311 [00:00<00:37, 7.33it/s, Materializing param=model.layers.2.self_attn.q_proj.weight] Loading weights: 11%|█ | 34/311 [00:00<00:37, 7.33it/s, Materializing param=model.layers.2.self_attn.q_proj.weight] Loading weights: 8%|▊ | 25/311 [00:00<00:20, 13.86it/s, Materializing param=model.layers.2.input_layernorm.weight] Loading weights: 5%|▌ | 17/311 [00:00<00:38, 7.59it/s, Materializing param=model.layers.1.mlp.up_proj.weight] Loading weights: 8%|▊ | 25/311 [00:00<00:20, 13.86it/s, Materializing param=model.layers.2.input_layernorm.weight] Loading weights: 5%|▌ | 17/311 [00:00<00:38, 7.59it/s, Materializing param=model.layers.1.mlp.up_proj.weight] Loading weights: 8%|▊ | 26/311 [00:00<00:20, 13.86it/s, Materializing param=model.layers.2.mlp.down_proj.weight] Loading weights: 8%|▊ | 26/311 [00:00<00:20, 13.86it/s, Materializing param=model.layers.2.mlp.down_proj.weight] Loading weights: 11%|█▏ | 35/311 [00:00<00:37, 7.33it/s, Materializing param=model.layers.2.self_attn.v_proj.weight] Loading weights: 11%|█▏ | 35/311 [00:00<00:37, 7.33it/s, Materializing param=model.layers.2.self_attn.v_proj.weight] Loading weights: 9%|▉ | 28/311 [00:00<00:39, 7.14it/s, Materializing param=model.layers.2.mlp.up_proj.weight] Loading weights: 9%|▉ | 28/311 [00:00<00:39, 7.14it/s, Materializing param=model.layers.2.mlp.up_proj.weight] Loading weights: 12%|█▏ | 36/311 [00:00<00:37, 7.33it/s, Materializing param=model.layers.3.input_layernorm.weight] Loading weights: 12%|█▏ | 36/311 [00:00<00:37, 7.33it/s, Materializing param=model.layers.3.input_layernorm.weight] Loading weights: 12%|█▏ | 37/311 [00:00<00:37, 7.33it/s, Materializing param=model.layers.3.mlp.down_proj.weight] Loading weights: 12%|█▏ | 37/311 [00:00<00:37, 7.33it/s, Materializing param=model.layers.3.mlp.down_proj.weight] Loading weights: 6%|▌ | 18/311 [00:00<00:38, 7.59it/s, Materializing param=model.layers.1.post_attention_layernorm.weight] Loading weights: 6%|▌ | 18/311 [00:00<00:38, 7.59it/s, Materializing param=model.layers.1.post_attention_layernorm.weight] Loading weights: 6%|▌ | 19/311 [00:00<00:38, 7.59it/s, Materializing param=model.layers.1.self_attn.k_norm.weight] Loading weights: 6%|▌ | 19/311 [00:00<00:38, 7.59it/s, Materializing param=model.layers.1.self_attn.k_norm.weight] Loading weights: 6%|▋ | 20/311 [00:00<00:38, 7.59it/s, Materializing param=model.layers.1.self_attn.k_proj.weight] Loading weights: 6%|▋ | 20/311 [00:00<00:38, 7.59it/s, Materializing param=model.layers.1.self_attn.k_proj.weight] Loading weights: 7%|▋ | 21/311 [00:00<00:38, 7.59it/s, Materializing param=model.layers.1.self_attn.o_proj.weight] Loading weights: 7%|▋ | 21/311 [00:00<00:38, 7.59it/s, Materializing param=model.layers.1.self_attn.o_proj.weight] Loading weights: 9%|▉ | 29/311 [00:00<00:39, 7.14it/s, Materializing param=model.layers.2.post_attention_layernorm.weight] Loading weights: 12%|█▏ | 38/311 [00:00<00:37, 7.33it/s, Materializing param=model.layers.3.mlp.gate_proj.weight] Loading weights: 12%|█▏ | 38/311 [00:00<00:37, 7.33it/s, Materializing param=model.layers.3.mlp.gate_proj.weight] Loading weights: 9%|▊ | 27/311 [00:00<00:20, 13.86it/s, Materializing param=model.layers.2.mlp.gate_proj.weight] Loading weights: 9%|▉ | 29/311 [00:00<00:39, 7.14it/s, Materializing param=model.layers.2.post_attention_layernorm.weight] Loading weights: 9%|▊ | 27/311 [00:00<00:20, 13.86it/s, Materializing param=model.layers.2.mlp.gate_proj.weight] Loading weights: 5%|▌ | 16/311 [00:00<00:46, 6.28it/s, Materializing param=model.layers.1.mlp.gate_proj.weight] Loading weights: 10%|▉ | 30/311 [00:00<00:39, 7.14it/s, Materializing param=model.layers.2.self_attn.k_norm.weight] Loading weights: 5%|▌ | 16/311 [00:00<00:46, 6.28it/s, Materializing param=model.layers.1.mlp.gate_proj.weight] Loading weights: 10%|▉ | 30/311 [00:00<00:39, 7.14it/s, Materializing param=model.layers.2.self_attn.k_norm.weight] Loading weights: 10%|▉ | 31/311 [00:00<00:39, 7.14it/s, Materializing param=model.layers.2.self_attn.k_proj.weight] Loading weights: 10%|▉ | 31/311 [00:00<00:39, 7.14it/s, Materializing param=model.layers.2.self_attn.k_proj.weight] Loading weights: 10%|█ | 32/311 [00:00<00:39, 7.14it/s, Materializing param=model.layers.2.self_attn.o_proj.weight] Loading weights: 10%|█ | 32/311 [00:00<00:39, 7.14it/s, Materializing param=model.layers.2.self_attn.o_proj.weight] Loading weights: 7%|▋ | 22/311 [00:00<00:38, 7.59it/s, Materializing param=model.layers.1.self_attn.q_norm.weight] Loading weights: 7%|▋ | 22/311 [00:00<00:38, 7.59it/s, Materializing param=model.layers.1.self_attn.q_norm.weight] Loading weights: 7%|▋ | 23/311 [00:00<00:37, 7.59it/s, Materializing param=model.layers.1.self_attn.q_proj.weight] Loading weights: 7%|▋ | 23/311 [00:00<00:37, 7.59it/s, Materializing param=model.layers.1.self_attn.q_proj.weight] Loading weights: 8%|▊ | 24/311 [00:00<00:37, 7.59it/s, Materializing param=model.layers.1.self_attn.v_proj.weight] Loading weights: 8%|▊ | 24/311 [00:00<00:37, 7.59it/s, Materializing param=model.layers.1.self_attn.v_proj.weight] Loading weights: 9%|▉ | 28/311 [00:00<00:20, 13.86it/s, Materializing param=model.layers.2.mlp.up_proj.weight] Loading weights: 9%|▉ | 28/311 [00:00<00:20, 13.86it/s, Materializing param=model.layers.2.mlp.up_proj.weight] Loading weights: 8%|▊ | 25/311 [00:00<00:37, 7.59it/s, Materializing param=model.layers.2.input_layernorm.weight] Loading weights: 8%|▊ | 25/311 [00:00<00:37, 7.59it/s, Materializing param=model.layers.2.input_layernorm.weight] Loading weights: 8%|▊ | 26/311 [00:00<00:37, 7.59it/s, Materializing param=model.layers.2.mlp.down_proj.weight] Loading weights: 8%|▊ | 26/311 [00:00<00:37, 7.59it/s, Materializing param=model.layers.2.mlp.down_proj.weight] Loading weights: 11%|█ | 33/311 [00:00<00:38, 7.14it/s, Materializing param=model.layers.2.self_attn.q_norm.weight] Loading weights: 11%|█ | 33/311 [00:00<00:38, 7.14it/s, Materializing param=model.layers.2.self_attn.q_norm.weight] Loading weights: 13%|█▎ | 39/311 [00:00<00:37, 7.33it/s, Materializing param=model.layers.3.mlp.up_proj.weight] Loading weights: 11%|█ | 34/311 [00:00<00:38, 7.14it/s, Materializing param=model.layers.2.self_attn.q_proj.weight] Loading weights: 11%|█ | 34/311 [00:00<00:38, 7.14it/s, Materializing param=model.layers.2.self_attn.q_proj.weight] Loading weights: 13%|█▎ | 39/311 [00:00<00:37, 7.33it/s, Materializing param=model.layers.3.mlp.up_proj.weight] Loading weights: 11%|█▏ | 35/311 [00:00<00:38, 7.14it/s, Materializing param=model.layers.2.self_attn.v_proj.weight] Loading weights: 11%|█▏ | 35/311 [00:00<00:38, 7.14it/s, Materializing param=model.layers.2.self_attn.v_proj.weight] Loading weights: 12%|█▏ | 36/311 [00:00<00:38, 7.14it/s, Materializing param=model.layers.3.input_layernorm.weight] Loading weights: 12%|█▏ | 36/311 [00:00<00:38, 7.14it/s, Materializing param=model.layers.3.input_layernorm.weight] Loading weights: 12%|█▏ | 37/311 [00:00<00:38, 7.14it/s, Materializing param=model.layers.3.mlp.down_proj.weight] Loading weights: 5%|▌ | 17/311 [00:00<00:46, 6.28it/s, Materializing param=model.layers.1.mlp.up_proj.weight] Loading weights: 12%|█▏ | 37/311 [00:00<00:38, 7.14it/s, Materializing param=model.layers.3.mlp.down_proj.weight] Loading weights: 5%|▌ | 17/311 [00:00<00:46, 6.28it/s, Materializing param=model.layers.1.mlp.up_proj.weight] Loading weights: 9%|▉ | 29/311 [00:00<00:20, 13.86it/s, Materializing param=model.layers.2.post_attention_layernorm.weight] Loading weights: 9%|▉ | 29/311 [00:00<00:20, 13.86it/s, Materializing param=model.layers.2.post_attention_layernorm.weight] Loading weights: 10%|▉ | 30/311 [00:00<00:20, 13.86it/s, Materializing param=model.layers.2.self_attn.k_norm.weight] Loading weights: 10%|▉ | 30/311 [00:00<00:20, 13.86it/s, Materializing param=model.layers.2.self_attn.k_norm.weight] Loading weights: 10%|▉ | 31/311 [00:00<00:20, 13.86it/s, Materializing param=model.layers.2.self_attn.k_proj.weight] Loading weights: 10%|▉ | 31/311 [00:00<00:20, 13.86it/s, Materializing param=model.layers.2.self_attn.k_proj.weight] Loading weights: 10%|█ | 32/311 [00:00<00:20, 13.86it/s, Materializing param=model.layers.2.self_attn.o_proj.weight] Loading weights: 10%|█ | 32/311 [00:00<00:20, 13.86it/s, Materializing param=model.layers.2.self_attn.o_proj.weight] Loading weights: 9%|▊ | 27/311 [00:00<00:37, 7.59it/s, Materializing param=model.layers.2.mlp.gate_proj.weight] Loading weights: 9%|▊ | 27/311 [00:00<00:37, 7.59it/s, Materializing param=model.layers.2.mlp.gate_proj.weight] Loading weights: 13%|█▎ | 40/311 [00:00<00:36, 7.33it/s, Materializing param=model.layers.3.post_attention_layernorm.weight] Loading weights: 13%|█▎ | 40/311 [00:00<00:36, 7.33it/s, Materializing param=model.layers.3.post_attention_layernorm.weight] Loading weights: 13%|█▎ | 41/311 [00:00<00:36, 7.33it/s, Materializing param=model.layers.3.self_attn.k_norm.weight] Loading weights: 13%|█▎ | 41/311 [00:00<00:36, 7.33it/s, Materializing param=model.layers.3.self_attn.k_norm.weight] Loading weights: 14%|█▎ | 42/311 [00:00<00:36, 7.33it/s, Materializing param=model.layers.3.self_attn.k_proj.weight] Loading weights: 14%|█▎ | 42/311 [00:00<00:36, 7.33it/s, Materializing param=model.layers.3.self_attn.k_proj.weight][INFO|configuration_utils.py:967] 2026-04-29 17:06:07,901 >> loading configuration file generation_config.json from cache at /root/.cache/huggingface/hub/models--Qwen--Qwen3-1.7B/snapshots/70d244cc86ccca08cf5af4e1e306ecf908b1ad5e/generation_config.json + Loading weights: 11%|█ | 33/311 [00:00<00:20, 13.86it/s, Materializing param=model.layers.2.self_attn.q_norm.weight] Loading weights: 14%|█▍ | 43/311 [00:00<00:36, 7.33it/s, Materializing param=model.layers.3.self_attn.o_proj.weight] Loading weights: 11%|█ | 33/311 [00:00<00:20, 13.86it/s, Materializing param=model.layers.2.self_attn.q_norm.weight] Loading weights: 14%|█▍ | 43/311 [00:00<00:36, 7.33it/s, Materializing param=model.layers.3.self_attn.o_proj.weight] Loading weights: 11%|█ | 34/311 [00:00<00:19, 13.86it/s, Materializing param=model.layers.2.self_attn.q_proj.weight][INFO|configuration_utils.py:1014] 2026-04-29 17:06:07,901 >> Generate config GenerationConfig { + "bos_token_id": 151643, + "do_sample": true, + "eos_token_id": [ + 151645, + 151643 + ], + "pad_token_id": 151643, + "temperature": 0.6, + "top_k": 20, + "top_p": 0.95 +} + + Loading weights: 11%|█ | 34/311 [00:00<00:19, 13.86it/s, Materializing param=model.layers.2.self_attn.q_proj.weight] Loading weights: 6%|▌ | 18/311 [00:00<00:46, 6.28it/s, Materializing param=model.layers.1.post_attention_layernorm.weight] Loading weights: 6%|▌ | 18/311 [00:00<00:46, 6.28it/s, Materializing param=model.layers.1.post_attention_layernorm.weight] Loading weights: 6%|▌ | 19/311 [00:00<00:46, 6.28it/s, Materializing param=model.layers.1.self_attn.k_norm.weight] Loading weights: 6%|▌ | 19/311 [00:00<00:46, 6.28it/s, Materializing param=model.layers.1.self_attn.k_norm.weight] Loading weights: 14%|█▍ | 44/311 [00:00<00:36, 7.33it/s, Materializing param=model.layers.3.self_attn.q_norm.weight] Loading weights: 14%|█▍ | 44/311 [00:00<00:36, 7.33it/s, Materializing param=model.layers.3.self_attn.q_norm.weight] Loading weights: 6%|▋ | 20/311 [00:00<00:46, 6.28it/s, Materializing param=model.layers.1.self_attn.k_proj.weight] Loading weights: 14%|█▍ | 45/311 [00:00<00:36, 7.33it/s, Materializing param=model.layers.3.self_attn.q_proj.weight] Loading weights: 6%|▋ | 20/311 [00:00<00:46, 6.28it/s, Materializing param=model.layers.1.self_attn.k_proj.weight] Loading weights: 14%|█▍ | 45/311 [00:00<00:36, 7.33it/s, Materializing param=model.layers.3.self_attn.q_proj.weight] Loading weights: 7%|▋ | 21/311 [00:00<00:46, 6.28it/s, Materializing param=model.layers.1.self_attn.o_proj.weight] Loading weights: 7%|▋ | 21/311 [00:00<00:46, 6.28it/s, Materializing param=model.layers.1.self_attn.o_proj.weight] Loading weights: 11%|█▏ | 35/311 [00:00<00:19, 13.86it/s, Materializing param=model.layers.2.self_attn.v_proj.weight] Loading weights: 11%|█▏ | 35/311 [00:00<00:19, 13.86it/s, Materializing param=model.layers.2.self_attn.v_proj.weight] Loading weights: 12%|█▏ | 36/311 [00:00<00:19, 13.86it/s, Materializing param=model.layers.3.input_layernorm.weight] Loading weights: 12%|█▏ | 36/311 [00:00<00:19, 13.86it/s, Materializing param=model.layers.3.input_layernorm.weight] Loading weights: 12%|█▏ | 37/311 [00:00<00:19, 13.86it/s, Materializing param=model.layers.3.mlp.down_proj.weight] Loading weights: 12%|█▏ | 37/311 [00:00<00:19, 13.86it/s, Materializing param=model.layers.3.mlp.down_proj.weight] Loading weights: 12%|█▏ | 38/311 [00:00<00:38, 7.14it/s, Materializing param=model.layers.3.mlp.gate_proj.weight] Loading weights: 12%|█▏ | 38/311 [00:00<00:38, 7.14it/s, Materializing param=model.layers.3.mlp.gate_proj.weight] Loading weights: 15%|█▍ | 46/311 [00:00<00:36, 7.33it/s, Materializing param=model.layers.3.self_attn.v_proj.weight] Loading weights: 15%|█▍ | 46/311 [00:00<00:36, 7.33it/s, Materializing param=model.layers.3.self_attn.v_proj.weight] Loading weights: 9%|▉ | 28/311 [00:00<00:37, 7.59it/s, Materializing param=model.layers.2.mlp.up_proj.weight] Loading weights: 1%| | 3/311 [00:00<00:23, 13.12it/s, Materializing param=model.layers.0.input_layernorm.weight] Loading weights: 1%| | 3/311 [00:00<00:23, 13.12it/s, Materializing param=model.layers.0.input_layernorm.weight] Loading weights: 9%|▉ | 28/311 [00:00<00:37, 7.59it/s, Materializing param=model.layers.2.mlp.up_proj.weight] Loading weights: 1%|▏ | 4/311 [00:00<00:26, 11.58it/s, Materializing param=model.layers.0.input_layernorm.weight] Loading weights: 1%|▏ | 4/311 [00:00<00:26, 11.58it/s, Materializing param=model.layers.0.mlp.down_proj.weight] Loading weights: 1%|▏ | 4/311 [00:00<00:26, 11.58it/s, Materializing param=model.layers.0.mlp.down_proj.weight] Loading weights: 15%|█▌ | 47/311 [00:00<00:01, 158.89it/s, Materializing param=model.layers.3.self_attn.v_proj.weight] Loading weights: 15%|█▌ | 47/311 [00:00<00:01, 158.89it/s, Materializing param=model.layers.4.input_layernorm.weight] Loading weights: 15%|█▌ | 47/311 [00:00<00:01, 158.89it/s, Materializing param=model.layers.4.input_layernorm.weight] Loading weights: 15%|█▌ | 48/311 [00:00<00:01, 158.89it/s, Materializing param=model.layers.4.mlp.down_proj.weight] Loading weights: 15%|█▌ | 48/311 [00:00<00:01, 158.89it/s, Materializing param=model.layers.4.mlp.down_proj.weight] Loading weights: 12%|█▏ | 38/311 [00:00<00:19, 13.86it/s, Materializing param=model.layers.3.mlp.gate_proj.weight] Loading weights: 12%|█▏ | 38/311 [00:00<00:19, 13.86it/s, Materializing param=model.layers.3.mlp.gate_proj.weight] Loading weights: 7%|▋ | 22/311 [00:00<00:46, 6.28it/s, Materializing param=model.layers.1.self_attn.q_norm.weight] Loading weights: 7%|▋ | 22/311 [00:00<00:46, 6.28it/s, Materializing param=model.layers.1.self_attn.q_norm.weight] Loading weights: 7%|▋ | 23/311 [00:00<00:45, 6.28it/s, Materializing param=model.layers.1.self_attn.q_proj.weight] Loading weights: 7%|▋ | 23/311 [00:00<00:45, 6.28it/s, Materializing param=model.layers.1.self_attn.q_proj.weight] Loading weights: 13%|█▎ | 39/311 [00:00<00:38, 7.14it/s, Materializing param=model.layers.3.mlp.up_proj.weight] Loading weights: 8%|▊ | 24/311 [00:00<00:45, 6.28it/s, Materializing param=model.layers.1.self_attn.v_proj.weight] Loading weights: 8%|▊ | 24/311 [00:00<00:45, 6.28it/s, Materializing param=model.layers.1.self_attn.v_proj.weight] Loading weights: 13%|█▎ | 39/311 [00:00<00:38, 7.14it/s, Materializing param=model.layers.3.mlp.up_proj.weight] Loading weights: 8%|▊ | 25/311 [00:00<00:45, 6.28it/s, Materializing param=model.layers.2.input_layernorm.weight] Loading weights: 8%|▊ | 25/311 [00:00<00:45, 6.28it/s, Materializing param=model.layers.2.input_layernorm.weight] Loading weights: 8%|▊ | 26/311 [00:00<00:45, 6.28it/s, Materializing param=model.layers.2.mlp.down_proj.weight] Loading weights: 8%|▊ | 26/311 [00:00<00:45, 6.28it/s, Materializing param=model.layers.2.mlp.down_proj.weight] Loading weights: 9%|▉ | 29/311 [00:00<00:37, 7.59it/s, Materializing param=model.layers.2.post_attention_layernorm.weight] Loading weights: 9%|▉ | 29/311 [00:00<00:37, 7.59it/s, Materializing param=model.layers.2.post_attention_layernorm.weight] Loading weights: 16%|█▌ | 49/311 [00:00<00:01, 158.89it/s, Materializing param=model.layers.4.mlp.gate_proj.weight] Loading weights: 16%|█▌ | 49/311 [00:00<00:01, 158.89it/s, Materializing param=model.layers.4.mlp.gate_proj.weight] Loading weights: 10%|▉ | 30/311 [00:00<00:37, 7.59it/s, Materializing param=model.layers.2.self_attn.k_norm.weight] Loading weights: 10%|▉ | 30/311 [00:00<00:37, 7.59it/s, Materializing param=model.layers.2.self_attn.k_norm.weight] Loading weights: 10%|▉ | 31/311 [00:00<00:36, 7.59it/s, Materializing param=model.layers.2.self_attn.k_proj.weight] Loading weights: 10%|▉ | 31/311 [00:00<00:36, 7.59it/s, Materializing param=model.layers.2.self_attn.k_proj.weight] Loading weights: 2%|▏ | 5/311 [00:00<00:26, 11.58it/s, Materializing param=model.layers.0.mlp.gate_proj.weight] Loading weights: 2%|▏ | 5/311 [00:00<00:26, 11.58it/s, Materializing param=model.layers.0.mlp.gate_proj.weight] Loading weights: 10%|█ | 32/311 [00:00<00:36, 7.59it/s, Materializing param=model.layers.2.self_attn.o_proj.weight] Loading weights: 10%|█ | 32/311 [00:00<00:36, 7.59it/s, Materializing param=model.layers.2.self_attn.o_proj.weight] Loading weights: 13%|█▎ | 39/311 [00:00<00:19, 13.86it/s, Materializing param=model.layers.3.mlp.up_proj.weight] Loading weights: 13%|█▎ | 39/311 [00:00<00:19, 13.86it/s, Materializing param=model.layers.3.mlp.up_proj.weight] Loading weights: 11%|█ | 33/311 [00:00<00:36, 7.59it/s, Materializing param=model.layers.2.self_attn.q_norm.weight] Loading weights: 11%|█ | 33/311 [00:00<00:36, 7.59it/s, Materializing param=model.layers.2.self_attn.q_norm.weight] Loading weights: 11%|█ | 34/311 [00:00<00:36, 7.59it/s, Materializing param=model.layers.2.self_attn.q_proj.weight] Loading weights: 11%|█ | 34/311 [00:00<00:36, 7.59it/s, Materializing param=model.layers.2.self_attn.q_proj.weight] Loading weights: 13%|█▎ | 40/311 [00:00<00:02, 132.29it/s, Materializing param=model.layers.3.mlp.up_proj.weight] Loading weights: 13%|█▎ | 40/311 [00:00<00:02, 132.29it/s, Materializing param=model.layers.3.post_attention_layernorm.weight] Loading weights: 13%|█▎ | 40/311 [00:00<00:02, 132.29it/s, Materializing param=model.layers.3.post_attention_layernorm.weight] Loading weights: 13%|█▎ | 41/311 [00:00<00:02, 132.29it/s, Materializing param=model.layers.3.self_attn.k_norm.weight] Loading weights: 13%|█▎ | 41/311 [00:00<00:02, 132.29it/s, Materializing param=model.layers.3.self_attn.k_norm.weight] Loading weights: 14%|█▎ | 42/311 [00:00<00:02, 132.29it/s, Materializing param=model.layers.3.self_attn.k_proj.weight] Loading weights: 14%|█▎ | 42/311 [00:00<00:02, 132.29it/s, Materializing param=model.layers.3.self_attn.k_proj.weight] Loading weights: 11%|█▏ | 35/311 [00:00<00:36, 7.59it/s, Materializing param=model.layers.2.self_attn.v_proj.weight] Loading weights: 14%|█▍ | 43/311 [00:00<00:02, 132.29it/s, Materializing param=model.layers.3.self_attn.o_proj.weight] Loading weights: 11%|█▏ | 35/311 [00:00<00:36, 7.59it/s, Materializing param=model.layers.2.self_attn.v_proj.weight] Loading weights: 14%|█▍ | 43/311 [00:00<00:02, 132.29it/s, Materializing param=model.layers.3.self_attn.o_proj.weight] Loading weights: 16%|█▌ | 50/311 [00:00<00:01, 158.89it/s, Materializing param=model.layers.4.mlp.up_proj.weight] Loading weights: 16%|█▌ | 50/311 [00:00<00:01, 158.89it/s, Materializing param=model.layers.4.mlp.up_proj.weight] Loading weights: 12%|█▏ | 36/311 [00:00<00:36, 7.59it/s, Materializing param=model.layers.3.input_layernorm.weight] Loading weights: 12%|█▏ | 36/311 [00:00<00:36, 7.59it/s, Materializing param=model.layers.3.input_layernorm.weight] Loading weights: 12%|█▏ | 37/311 [00:00<00:36, 7.59it/s, Materializing param=model.layers.3.mlp.down_proj.weight] Loading weights: 12%|█▏ | 37/311 [00:00<00:36, 7.59it/s, Materializing param=model.layers.3.mlp.down_proj.weight] Loading weights: 14%|█▍ | 44/311 [00:00<00:02, 132.29it/s, Materializing param=model.layers.3.self_attn.q_norm.weight] Loading weights: 14%|█▍ | 44/311 [00:00<00:02, 132.29it/s, Materializing param=model.layers.3.self_attn.q_norm.weight] Loading weights: 13%|█▎ | 40/311 [00:00<00:19, 13.86it/s, Materializing param=model.layers.3.post_attention_layernorm.weight] Loading weights: 13%|█▎ | 40/311 [00:00<00:19, 13.86it/s, Materializing param=model.layers.3.post_attention_layernorm.weight] Loading weights: 13%|█▎ | 41/311 [00:00<00:19, 13.86it/s, Materializing param=model.layers.3.self_attn.k_norm.weight] Loading weights: 13%|█▎ | 41/311 [00:00<00:19, 13.86it/s, Materializing param=model.layers.3.self_attn.k_norm.weight] Loading weights: 14%|█▎ | 42/311 [00:00<00:19, 13.86it/s, Materializing param=model.layers.3.self_attn.k_proj.weight] Loading weights: 14%|█▎ | 42/311 [00:00<00:19, 13.86it/s, Materializing param=model.layers.3.self_attn.k_proj.weight] Loading weights: 14%|█▍ | 43/311 [00:00<00:19, 13.86it/s, Materializing param=model.layers.3.self_attn.o_proj.weight] Loading weights: 14%|█▍ | 43/311 [00:00<00:19, 13.86it/s, Materializing param=model.layers.3.self_attn.o_proj.weight] Loading weights: 14%|█▍ | 44/311 [00:00<00:19, 13.86it/s, Materializing param=model.layers.3.self_attn.q_norm.weight] Loading weights: 14%|█▍ | 44/311 [00:00<00:19, 13.86it/s, Materializing param=model.layers.3.self_attn.q_norm.weight] Loading weights: 12%|█▏ | 38/311 [00:00<00:35, 7.59it/s, Materializing param=model.layers.3.mlp.gate_proj.weight] Loading weights: 12%|█▏ | 38/311 [00:00<00:35, 7.59it/s, Materializing param=model.layers.3.mlp.gate_proj.weight] Loading weights: 16%|█▋ | 51/311 [00:00<00:01, 158.89it/s, Materializing param=model.layers.4.post_attention_layernorm.weight] Loading weights: 16%|█▋ | 51/311 [00:00<00:01, 158.89it/s, Materializing param=model.layers.4.post_attention_layernorm.weight] Loading weights: 17%|█▋ | 52/311 [00:00<00:01, 158.89it/s, Materializing param=model.layers.4.self_attn.k_norm.weight] Loading weights: 17%|█▋ | 52/311 [00:00<00:01, 158.89it/s, Materializing param=model.layers.4.self_attn.k_norm.weight] Loading weights: 17%|█▋ | 53/311 [00:00<00:01, 158.89it/s, Materializing param=model.layers.4.self_attn.k_proj.weight] Loading weights: 17%|█▋ | 53/311 [00:00<00:01, 158.89it/s, Materializing param=model.layers.4.self_attn.k_proj.weight] Loading weights: 14%|█▍ | 45/311 [00:00<00:19, 13.86it/s, Materializing param=model.layers.3.self_attn.q_proj.weight] Loading weights: 14%|█▍ | 45/311 [00:00<00:19, 13.86it/s, Materializing param=model.layers.3.self_attn.q_proj.weight] Loading weights: 15%|█▍ | 46/311 [00:00<00:19, 13.86it/s, Materializing param=model.layers.3.self_attn.v_proj.weight] Loading weights: 15%|█▍ | 46/311 [00:00<00:19, 13.86it/s, Materializing param=model.layers.3.self_attn.v_proj.weight] Loading weights: 17%|█▋ | 54/311 [00:00<00:01, 158.89it/s, Materializing param=model.layers.4.self_attn.o_proj.weight] Loading weights: 17%|█▋ | 54/311 [00:00<00:01, 158.89it/s, Materializing param=model.layers.4.self_attn.o_proj.weight] Loading weights: 15%|█▌ | 47/311 [00:00<00:19, 13.86it/s, Materializing param=model.layers.4.input_layernorm.weight] Loading weights: 15%|█▌ | 47/311 [00:00<00:19, 13.86it/s, Materializing param=model.layers.4.input_layernorm.weight] Loading weights: 15%|█▌ | 48/311 [00:00<00:18, 13.86it/s, Materializing param=model.layers.4.mlp.down_proj.weight] Loading weights: 15%|█▌ | 48/311 [00:00<00:18, 13.86it/s, Materializing param=model.layers.4.mlp.down_proj.weight] Loading weights: 2%|▏ | 6/311 [00:00<00:26, 11.58it/s, Materializing param=model.layers.0.mlp.up_proj.weight] Loading weights: 2%|▏ | 6/311 [00:00<00:26, 11.58it/s, Materializing param=model.layers.0.mlp.up_proj.weight] Loading weights: 2%|▏ | 7/311 [00:00<00:26, 11.58it/s, Materializing param=model.layers.0.post_attention_layernorm.weight] Loading weights: 2%|▏ | 7/311 [00:00<00:26, 11.58it/s, Materializing param=model.layers.0.post_attention_layernorm.weight] Loading weights: 18%|█▊ | 55/311 [00:00<00:01, 158.89it/s, Materializing param=model.layers.4.self_attn.q_norm.weight] Loading weights: 18%|█▊ | 55/311 [00:00<00:01, 158.89it/s, Materializing param=model.layers.4.self_attn.q_norm.weight] Loading weights: 3%|▎ | 8/311 [00:00<00:26, 11.58it/s, Materializing param=model.layers.0.self_attn.k_norm.weight] Loading weights: 3%|▎ | 8/311 [00:00<00:26, 11.58it/s, Materializing param=model.layers.0.self_attn.k_norm.weight] Loading weights: 18%|█▊ | 56/311 [00:00<00:01, 158.89it/s, Materializing param=model.layers.4.self_attn.q_proj.weight] Loading weights: 18%|█▊ | 56/311 [00:00<00:01, 158.89it/s, Materializing param=model.layers.4.self_attn.q_proj.weight] Loading weights: 3%|▎ | 9/311 [00:00<00:26, 11.58it/s, Materializing param=model.layers.0.self_attn.k_proj.weight] Loading weights: 3%|▎ | 9/311 [00:00<00:26, 11.58it/s, Materializing param=model.layers.0.self_attn.k_proj.weight] Loading weights: 3%|▎ | 10/311 [00:00<00:26, 11.58it/s, Materializing param=model.layers.0.self_attn.o_proj.weight] Loading weights: 3%|▎ | 10/311 [00:00<00:26, 11.58it/s, Materializing param=model.layers.0.self_attn.o_proj.weight] Loading weights: 9%|▊ | 27/311 [00:00<00:45, 6.28it/s, Materializing param=model.layers.2.mlp.gate_proj.weight] Loading weights: 9%|▊ | 27/311 [00:00<00:45, 6.28it/s, Materializing param=model.layers.2.mlp.gate_proj.weight] Loading weights: 9%|▉ | 28/311 [00:00<00:45, 6.28it/s, Materializing param=model.layers.2.mlp.up_proj.weight] Loading weights: 9%|▉ | 28/311 [00:00<00:45, 6.28it/s, Materializing param=model.layers.2.mlp.up_proj.weight] Loading weights: 18%|█▊ | 57/311 [00:00<00:01, 158.89it/s, Materializing param=model.layers.4.self_attn.v_proj.weight] Loading weights: 14%|█▍ | 45/311 [00:00<00:02, 132.29it/s, Materializing param=model.layers.3.self_attn.q_proj.weight] Loading weights: 18%|█▊ | 57/311 [00:00<00:01, 158.89it/s, Materializing param=model.layers.4.self_attn.v_proj.weight] Loading weights: 14%|█▍ | 45/311 [00:00<00:02, 132.29it/s, Materializing param=model.layers.3.self_attn.q_proj.weight] Loading weights: 15%|█▍ | 46/311 [00:00<00:02, 132.29it/s, Materializing param=model.layers.3.self_attn.v_proj.weight] Loading weights: 15%|█▍ | 46/311 [00:00<00:02, 132.29it/s, Materializing param=model.layers.3.self_attn.v_proj.weight] Loading weights: 15%|█▌ | 47/311 [00:00<00:01, 132.29it/s, Materializing param=model.layers.4.input_layernorm.weight] Loading weights: 16%|█▌ | 49/311 [00:00<00:18, 13.86it/s, Materializing param=model.layers.4.mlp.gate_proj.weight] Loading weights: 15%|█▌ | 47/311 [00:00<00:01, 132.29it/s, Materializing param=model.layers.4.input_layernorm.weight] Loading weights: 16%|█▌ | 49/311 [00:00<00:18, 13.86it/s, Materializing param=model.layers.4.mlp.gate_proj.weight] Loading weights: 15%|█▌ | 48/311 [00:00<00:01, 132.29it/s, Materializing param=model.layers.4.mlp.down_proj.weight] Loading weights: 15%|█▌ | 48/311 [00:00<00:01, 132.29it/s, Materializing param=model.layers.4.mlp.down_proj.weight] Loading weights: 16%|█▌ | 49/311 [00:00<00:01, 132.29it/s, Materializing param=model.layers.4.mlp.gate_proj.weight] Loading weights: 19%|█▊ | 58/311 [00:00<00:01, 158.89it/s, Materializing param=model.layers.5.input_layernorm.weight] Loading weights: 19%|█▊ | 58/311 [00:00<00:01, 158.89it/s, Materializing param=model.layers.5.input_layernorm.weight] Loading weights: 16%|█▌ | 49/311 [00:00<00:01, 132.29it/s, Materializing param=model.layers.4.mlp.gate_proj.weight] Loading weights: 4%|▎ | 11/311 [00:00<00:25, 11.58it/s, Materializing param=model.layers.0.self_attn.q_norm.weight] Loading weights: 4%|▎ | 11/311 [00:00<00:25, 11.58it/s, Materializing param=model.layers.0.self_attn.q_norm.weight] Loading weights: 4%|▍ | 12/311 [00:00<00:25, 11.58it/s, Materializing param=model.layers.0.self_attn.q_proj.weight] Loading weights: 19%|█▉ | 59/311 [00:00<00:01, 158.89it/s, Materializing param=model.layers.5.mlp.down_proj.weight] Loading weights: 4%|▍ | 12/311 [00:00<00:25, 11.58it/s, Materializing param=model.layers.0.self_attn.q_proj.weight] Loading weights: 19%|█▉ | 59/311 [00:00<00:01, 158.89it/s, Materializing param=model.layers.5.mlp.down_proj.weight] Loading weights: 13%|█▎ | 39/311 [00:00<00:35, 7.59it/s, Materializing param=model.layers.3.mlp.up_proj.weight] Loading weights: 13%|█▎ | 39/311 [00:00<00:35, 7.59it/s, Materializing param=model.layers.3.mlp.up_proj.weight] Loading weights: 4%|▍ | 13/311 [00:00<00:25, 11.58it/s, Materializing param=model.layers.0.self_attn.v_proj.weight] Loading weights: 4%|▍ | 13/311 [00:00<00:25, 11.58it/s, Materializing param=model.layers.0.self_attn.v_proj.weight] Loading weights: 5%|▍ | 14/311 [00:00<00:25, 11.58it/s, Materializing param=model.layers.1.input_layernorm.weight] Loading weights: 5%|▍ | 14/311 [00:00<00:25, 11.58it/s, Materializing param=model.layers.1.input_layernorm.weight] Loading weights: 5%|▍ | 15/311 [00:00<00:25, 11.58it/s, Materializing param=model.layers.1.mlp.down_proj.weight] Loading weights: 5%|▍ | 15/311 [00:00<00:25, 11.58it/s, Materializing param=model.layers.1.mlp.down_proj.weight] Loading weights: 19%|█▉ | 60/311 [00:00<00:01, 158.89it/s, Materializing param=model.layers.5.mlp.gate_proj.weight] Loading weights: 19%|█▉ | 60/311 [00:00<00:01, 158.89it/s, Materializing param=model.layers.5.mlp.gate_proj.weight] Loading weights: 16%|█▌ | 50/311 [00:00<00:01, 132.29it/s, Materializing param=model.layers.4.mlp.up_proj.weight] Loading weights: 9%|▉ | 29/311 [00:00<00:44, 6.28it/s, Materializing param=model.layers.2.post_attention_layernorm.weight] Loading weights: 16%|█▌ | 50/311 [00:00<00:01, 132.29it/s, Materializing param=model.layers.4.mlp.up_proj.weight] Loading weights: 9%|▉ | 29/311 [00:00<00:44, 6.28it/s, Materializing param=model.layers.2.post_attention_layernorm.weight] Loading weights: 10%|▉ | 30/311 [00:00<00:44, 6.28it/s, Materializing param=model.layers.2.self_attn.k_norm.weight] Loading weights: 10%|▉ | 30/311 [00:00<00:44, 6.28it/s, Materializing param=model.layers.2.self_attn.k_norm.weight] Loading weights: 10%|▉ | 31/311 [00:00<00:44, 6.28it/s, Materializing param=model.layers.2.self_attn.k_proj.weight] Loading weights: 16%|█▌ | 50/311 [00:00<00:01, 175.84it/s, Materializing param=model.layers.4.mlp.gate_proj.weight] Loading weights: 10%|▉ | 31/311 [00:00<00:44, 6.28it/s, Materializing param=model.layers.2.self_attn.k_proj.weight] Loading weights: 16%|█▌ | 50/311 [00:00<00:01, 175.84it/s, Materializing param=model.layers.4.mlp.up_proj.weight] Loading weights: 16%|█▌ | 50/311 [00:00<00:01, 175.84it/s, Materializing param=model.layers.4.mlp.up_proj.weight] Loading weights: 13%|█▎ | 40/311 [00:00<00:35, 7.59it/s, Materializing param=model.layers.3.post_attention_layernorm.weight] Loading weights: 13%|█▎ | 40/311 [00:00<00:35, 7.59it/s, Materializing param=model.layers.3.post_attention_layernorm.weight] Loading weights: 10%|█ | 32/311 [00:00<00:44, 6.28it/s, Materializing param=model.layers.2.self_attn.o_proj.weight] Loading weights: 13%|█▎ | 41/311 [00:00<00:35, 7.59it/s, Materializing param=model.layers.3.self_attn.k_norm.weight] Loading weights: 10%|█ | 32/311 [00:00<00:44, 6.28it/s, Materializing param=model.layers.2.self_attn.o_proj.weight] Loading weights: 13%|█▎ | 41/311 [00:00<00:35, 7.59it/s, Materializing param=model.layers.3.self_attn.k_norm.weight] Loading weights: 14%|█▎ | 42/311 [00:00<00:35, 7.59it/s, Materializing param=model.layers.3.self_attn.k_proj.weight] Loading weights: 14%|█▎ | 42/311 [00:00<00:35, 7.59it/s, Materializing param=model.layers.3.self_attn.k_proj.weight] Loading weights: 14%|█▍ | 43/311 [00:00<00:35, 7.59it/s, Materializing param=model.layers.3.self_attn.o_proj.weight] Loading weights: 14%|█▍ | 43/311 [00:00<00:35, 7.59it/s, Materializing param=model.layers.3.self_attn.o_proj.weight] Loading weights: 14%|█▍ | 44/311 [00:00<00:35, 7.59it/s, Materializing param=model.layers.3.self_attn.q_norm.weight] Loading weights: 14%|█▍ | 44/311 [00:00<00:35, 7.59it/s, Materializing param=model.layers.3.self_attn.q_norm.weight] Loading weights: 16%|█▋ | 51/311 [00:00<00:01, 175.84it/s, Materializing param=model.layers.4.post_attention_layernorm.weight] Loading weights: 16%|█▋ | 51/311 [00:00<00:01, 175.84it/s, Materializing param=model.layers.4.post_attention_layernorm.weight] Loading weights: 11%|█ | 33/311 [00:00<00:44, 6.28it/s, Materializing param=model.layers.2.self_attn.q_norm.weight] Loading weights: 11%|█ | 33/311 [00:00<00:44, 6.28it/s, Materializing param=model.layers.2.self_attn.q_norm.weight] Loading weights: 17%|█▋ | 52/311 [00:00<00:01, 175.84it/s, Materializing param=model.layers.4.self_attn.k_norm.weight] Loading weights: 17%|█▋ | 52/311 [00:00<00:01, 175.84it/s, Materializing param=model.layers.4.self_attn.k_norm.weight] Loading weights: 11%|█ | 34/311 [00:00<00:44, 6.28it/s, Materializing param=model.layers.2.self_attn.q_proj.weight] Loading weights: 11%|█ | 34/311 [00:00<00:44, 6.28it/s, Materializing param=model.layers.2.self_attn.q_proj.weight] Loading weights: 20%|█▉ | 61/311 [00:00<00:01, 158.89it/s, Materializing param=model.layers.5.mlp.up_proj.weight] Loading weights: 20%|█▉ | 61/311 [00:00<00:01, 158.89it/s, Materializing param=model.layers.5.mlp.up_proj.weight] Loading weights: 17%|█▋ | 53/311 [00:00<00:01, 175.84it/s, Materializing param=model.layers.4.self_attn.k_proj.weight] Loading weights: 17%|█▋ | 53/311 [00:00<00:01, 175.84it/s, Materializing param=model.layers.4.self_attn.k_proj.weight] Loading weights: 5%|▌ | 16/311 [00:00<00:25, 11.58it/s, Materializing param=model.layers.1.mlp.gate_proj.weight] Loading weights: 17%|█▋ | 54/311 [00:00<00:01, 175.84it/s, Materializing param=model.layers.4.self_attn.o_proj.weight] Loading weights: 17%|█▋ | 54/311 [00:00<00:01, 175.84it/s, Materializing param=model.layers.4.self_attn.o_proj.weight] Loading weights: 5%|▌ | 16/311 [00:00<00:25, 11.58it/s, Materializing param=model.layers.1.mlp.gate_proj.weight] Loading weights: 16%|█▋ | 51/311 [00:00<00:01, 132.29it/s, Materializing param=model.layers.4.post_attention_layernorm.weight] Loading weights: 16%|█▋ | 51/311 [00:00<00:01, 132.29it/s, Materializing param=model.layers.4.post_attention_layernorm.weight] Loading weights: 17%|█▋ | 52/311 [00:00<00:01, 132.29it/s, Materializing param=model.layers.4.self_attn.k_norm.weight] Loading weights: 17%|█▋ | 52/311 [00:00<00:01, 132.29it/s, Materializing param=model.layers.4.self_attn.k_norm.weight] Loading weights: 17%|█▋ | 53/311 [00:00<00:01, 132.29it/s, Materializing param=model.layers.4.self_attn.k_proj.weight] Loading weights: 14%|█▍ | 45/311 [00:00<00:01, 155.68it/s, Materializing param=model.layers.3.self_attn.q_norm.weight] Loading weights: 14%|█▍ | 45/311 [00:00<00:01, 155.68it/s, Materializing param=model.layers.3.self_attn.q_proj.weight] Loading weights: 17%|█▋ | 53/311 [00:00<00:01, 132.29it/s, Materializing param=model.layers.4.self_attn.k_proj.weight] Loading weights: 14%|█▍ | 45/311 [00:00<00:01, 155.68it/s, Materializing param=model.layers.3.self_attn.q_proj.weight] Loading weights: 15%|█▍ | 46/311 [00:00<00:01, 155.68it/s, Materializing param=model.layers.3.self_attn.v_proj.weight] Loading weights: 15%|█▍ | 46/311 [00:00<00:01, 155.68it/s, Materializing param=model.layers.3.self_attn.v_proj.weight] Loading weights: 17%|█▋ | 54/311 [00:00<00:01, 132.29it/s, Materializing param=model.layers.4.self_attn.o_proj.weight] Loading weights: 15%|█▌ | 47/311 [00:00<00:01, 155.68it/s, Materializing param=model.layers.4.input_layernorm.weight] Loading weights: 18%|█▊ | 55/311 [00:00<00:01, 175.84it/s, Materializing param=model.layers.4.self_attn.q_norm.weight] Loading weights: 17%|█▋ | 54/311 [00:00<00:01, 132.29it/s, Materializing param=model.layers.4.self_attn.o_proj.weight] Loading weights: 15%|█▌ | 47/311 [00:00<00:01, 155.68it/s, Materializing param=model.layers.4.input_layernorm.weight] Loading weights: 18%|█▊ | 55/311 [00:00<00:01, 175.84it/s, Materializing param=model.layers.4.self_attn.q_norm.weight] Loading weights: 15%|█▌ | 48/311 [00:00<00:01, 155.68it/s, Materializing param=model.layers.4.mlp.down_proj.weight] Loading weights: 15%|█▌ | 48/311 [00:00<00:01, 155.68it/s, Materializing param=model.layers.4.mlp.down_proj.weight] Loading weights: 18%|█▊ | 56/311 [00:00<00:01, 175.84it/s, Materializing param=model.layers.4.self_attn.q_proj.weight] Loading weights: 18%|█▊ | 56/311 [00:00<00:01, 175.84it/s, Materializing param=model.layers.4.self_attn.q_proj.weight] Loading weights: 11%|█▏ | 35/311 [00:00<00:43, 6.28it/s, Materializing param=model.layers.2.self_attn.v_proj.weight] Loading weights: 11%|█▏ | 35/311 [00:00<00:43, 6.28it/s, Materializing param=model.layers.2.self_attn.v_proj.weight] Loading weights: 12%|█▏ | 36/311 [00:00<00:43, 6.28it/s, Materializing param=model.layers.3.input_layernorm.weight] Loading weights: 18%|█▊ | 57/311 [00:00<00:01, 175.84it/s, Materializing param=model.layers.4.self_attn.v_proj.weight] Loading weights: 12%|█▏ | 36/311 [00:00<00:43, 6.28it/s, Materializing param=model.layers.3.input_layernorm.weight] Loading weights: 18%|█▊ | 57/311 [00:00<00:01, 175.84it/s, Materializing param=model.layers.4.self_attn.v_proj.weight] Loading weights: 12%|█▏ | 37/311 [00:00<00:43, 6.28it/s, Materializing param=model.layers.3.mlp.down_proj.weight] Loading weights: 12%|█▏ | 37/311 [00:00<00:43, 6.28it/s, Materializing param=model.layers.3.mlp.down_proj.weight] Loading weights: 18%|█▊ | 55/311 [00:00<00:01, 132.29it/s, Materializing param=model.layers.4.self_attn.q_norm.weight] Loading weights: 18%|█▊ | 55/311 [00:00<00:01, 132.29it/s, Materializing param=model.layers.4.self_attn.q_norm.weight] Loading weights: 18%|█▊ | 56/311 [00:00<00:01, 132.29it/s, Materializing param=model.layers.4.self_attn.q_proj.weight] Loading weights: 18%|█▊ | 56/311 [00:00<00:01, 132.29it/s, Materializing param=model.layers.4.self_attn.q_proj.weight] Loading weights: 19%|█▊ | 58/311 [00:00<00:01, 175.84it/s, Materializing param=model.layers.5.input_layernorm.weight] Loading weights: 19%|█▊ | 58/311 [00:00<00:01, 175.84it/s, Materializing param=model.layers.5.input_layernorm.weight] Loading weights: 19%|█▉ | 59/311 [00:00<00:01, 175.84it/s, Materializing param=model.layers.5.mlp.down_proj.weight] Loading weights: 19%|█▉ | 59/311 [00:00<00:01, 175.84it/s, Materializing param=model.layers.5.mlp.down_proj.weight] Loading weights: 18%|█▊ | 57/311 [00:00<00:01, 132.29it/s, Materializing param=model.layers.4.self_attn.v_proj.weight] Loading weights: 18%|█▊ | 57/311 [00:00<00:01, 132.29it/s, Materializing param=model.layers.4.self_attn.v_proj.weight] Loading weights: 20%|█▉ | 62/311 [00:00<00:01, 158.89it/s, Materializing param=model.layers.5.post_attention_layernorm.weight] Loading weights: 20%|█▉ | 62/311 [00:00<00:01, 158.89it/s, Materializing param=model.layers.5.post_attention_layernorm.weight] Loading weights: 20%|██ | 63/311 [00:00<00:01, 158.89it/s, Materializing param=model.layers.5.self_attn.k_norm.weight] Loading weights: 20%|██ | 63/311 [00:00<00:01, 158.89it/s, Materializing param=model.layers.5.self_attn.k_norm.weight] Loading weights: 21%|██ | 64/311 [00:00<00:01, 158.89it/s, Materializing param=model.layers.5.self_attn.k_proj.weight] Loading weights: 21%|██ | 64/311 [00:00<00:01, 158.89it/s, Materializing param=model.layers.5.self_attn.k_proj.weight] Loading weights: 19%|█▊ | 58/311 [00:00<00:01, 132.29it/s, Materializing param=model.layers.5.input_layernorm.weight] Loading weights: 21%|██ | 65/311 [00:00<00:01, 158.89it/s, Materializing param=model.layers.5.self_attn.o_proj.weight] Loading weights: 19%|█▊ | 58/311 [00:00<00:01, 132.29it/s, Materializing param=model.layers.5.input_layernorm.weight] Loading weights: 21%|██ | 65/311 [00:00<00:01, 158.89it/s, Materializing param=model.layers.5.self_attn.o_proj.weight] Loading weights: 19%|█▉ | 59/311 [00:00<00:01, 132.29it/s, Materializing param=model.layers.5.mlp.down_proj.weight] Loading weights: 19%|█▉ | 59/311 [00:00<00:01, 132.29it/s, Materializing param=model.layers.5.mlp.down_proj.weight] Loading weights: 5%|▌ | 17/311 [00:00<00:25, 11.58it/s, Materializing param=model.layers.1.mlp.up_proj.weight] Loading weights: 5%|▌ | 17/311 [00:00<00:25, 11.58it/s, Materializing param=model.layers.1.mlp.up_proj.weight] Loading weights: 19%|█▉ | 60/311 [00:00<00:01, 175.84it/s, Materializing param=model.layers.5.mlp.gate_proj.weight] Loading weights: 19%|█▉ | 60/311 [00:00<00:01, 175.84it/s, Materializing param=model.layers.5.mlp.gate_proj.weight] Loading weights: 21%|██ | 66/311 [00:00<00:01, 158.89it/s, Materializing param=model.layers.5.self_attn.q_norm.weight] Loading weights: 21%|██ | 66/311 [00:00<00:01, 158.89it/s, Materializing param=model.layers.5.self_attn.q_norm.weight] Loading weights: 22%|██▏ | 67/311 [00:00<00:01, 158.89it/s, Materializing param=model.layers.5.self_attn.q_proj.weight] Loading weights: 22%|██▏ | 67/311 [00:00<00:01, 158.89it/s, Materializing param=model.layers.5.self_attn.q_proj.weight] Loading weights: 12%|█▏ | 38/311 [00:00<00:02, 114.67it/s, Materializing param=model.layers.3.mlp.down_proj.weight] Loading weights: 12%|█▏ | 38/311 [00:00<00:02, 114.67it/s, Materializing param=model.layers.3.mlp.gate_proj.weight] Loading weights: 12%|█▏ | 38/311 [00:00<00:02, 114.67it/s, Materializing param=model.layers.3.mlp.gate_proj.weight] Loading weights: 22%|██▏ | 68/311 [00:00<00:01, 158.89it/s, Materializing param=model.layers.5.self_attn.v_proj.weight] Loading weights: 22%|██▏ | 68/311 [00:00<00:01, 158.89it/s, Materializing param=model.layers.5.self_attn.v_proj.weight] Loading weights: 22%|██▏ | 69/311 [00:00<00:01, 158.89it/s, Materializing param=model.layers.6.input_layernorm.weight] Loading weights: 22%|██▏ | 69/311 [00:00<00:01, 158.89it/s, Materializing param=model.layers.6.input_layernorm.weight] Loading weights: 23%|██▎ | 70/311 [00:00<00:01, 158.89it/s, Materializing param=model.layers.6.mlp.down_proj.weight] Loading weights: 23%|██▎ | 70/311 [00:00<00:01, 158.89it/s, Materializing param=model.layers.6.mlp.down_proj.weight] Loading weights: 19%|█▉ | 60/311 [00:00<00:01, 132.29it/s, Materializing param=model.layers.5.mlp.gate_proj.weight] Loading weights: 19%|█▉ | 60/311 [00:00<00:01, 132.29it/s, Materializing param=model.layers.5.mlp.gate_proj.weight] Loading weights: 16%|█▌ | 49/311 [00:00<00:01, 155.68it/s, Materializing param=model.layers.4.mlp.gate_proj.weight] Loading weights: 16%|█▌ | 49/311 [00:00<00:01, 155.68it/s, Materializing param=model.layers.4.mlp.gate_proj.weight] Loading weights: 16%|█▌ | 50/311 [00:00<00:01, 155.68it/s, Materializing param=model.layers.4.mlp.up_proj.weight] Loading weights: 16%|█▌ | 50/311 [00:00<00:01, 155.68it/s, Materializing param=model.layers.4.mlp.up_proj.weight] Loading weights: 20%|█▉ | 61/311 [00:00<00:01, 175.84it/s, Materializing param=model.layers.5.mlp.up_proj.weight] Loading weights: 20%|█▉ | 61/311 [00:00<00:01, 175.84it/s, Materializing param=model.layers.5.mlp.up_proj.weight] Loading weights: 6%|▌ | 18/311 [00:00<00:25, 11.58it/s, Materializing param=model.layers.1.post_attention_layernorm.weight] Loading weights: 6%|▌ | 18/311 [00:00<00:25, 11.58it/s, Materializing param=model.layers.1.post_attention_layernorm.weight] Loading weights: 6%|▌ | 19/311 [00:00<00:25, 11.58it/s, Materializing param=model.layers.1.self_attn.k_norm.weight] Loading weights: 6%|▌ | 19/311 [00:00<00:25, 11.58it/s, Materializing param=model.layers.1.self_attn.k_norm.weight] Loading weights: 6%|▋ | 20/311 [00:00<00:25, 11.58it/s, Materializing param=model.layers.1.self_attn.k_proj.weight] Loading weights: 6%|▋ | 20/311 [00:00<00:25, 11.58it/s, Materializing param=model.layers.1.self_attn.k_proj.weight] Loading weights: 7%|▋ | 21/311 [00:00<00:25, 11.58it/s, Materializing param=model.layers.1.self_attn.o_proj.weight] Loading weights: 7%|▋ | 21/311 [00:00<00:25, 11.58it/s, Materializing param=model.layers.1.self_attn.o_proj.weight] Loading weights: 13%|█▎ | 39/311 [00:00<00:02, 114.67it/s, Materializing param=model.layers.3.mlp.up_proj.weight] Loading weights: 13%|█▎ | 39/311 [00:00<00:02, 114.67it/s, Materializing param=model.layers.3.mlp.up_proj.weight] Loading weights: 23%|██▎ | 71/311 [00:00<00:01, 158.89it/s, Materializing param=model.layers.6.mlp.gate_proj.weight] Loading weights: 23%|██▎ | 71/311 [00:00<00:01, 158.89it/s, Materializing param=model.layers.6.mlp.gate_proj.weight] Loading weights: 16%|█▋ | 51/311 [00:00<00:01, 155.68it/s, Materializing param=model.layers.4.post_attention_layernorm.weight] Loading weights: 20%|█▉ | 61/311 [00:00<00:01, 132.29it/s, Materializing param=model.layers.5.mlp.up_proj.weight] Loading weights: 16%|█▋ | 51/311 [00:00<00:01, 155.68it/s, Materializing param=model.layers.4.post_attention_layernorm.weight] Loading weights: 20%|█▉ | 61/311 [00:00<00:01, 132.29it/s, Materializing param=model.layers.5.mlp.up_proj.weight] Loading weights: 17%|█▋ | 52/311 [00:00<00:01, 155.68it/s, Materializing param=model.layers.4.self_attn.k_norm.weight] Loading weights: 7%|▋ | 22/311 [00:00<00:24, 11.58it/s, Materializing param=model.layers.1.self_attn.q_norm.weight] Loading weights: 17%|█▋ | 52/311 [00:00<00:01, 155.68it/s, Materializing param=model.layers.4.self_attn.k_norm.weight] Loading weights: 7%|▋ | 22/311 [00:00<00:24, 11.58it/s, Materializing param=model.layers.1.self_attn.q_norm.weight] Loading weights: 17%|█▋ | 53/311 [00:00<00:01, 155.68it/s, Materializing param=model.layers.4.self_attn.k_proj.weight] Loading weights: 17%|█▋ | 53/311 [00:00<00:01, 155.68it/s, Materializing param=model.layers.4.self_attn.k_proj.weight] Loading weights: 20%|█▉ | 62/311 [00:00<00:01, 175.84it/s, Materializing param=model.layers.5.post_attention_layernorm.weight] Loading weights: 20%|█▉ | 62/311 [00:00<00:01, 175.84it/s, Materializing param=model.layers.5.post_attention_layernorm.weight] Loading weights: 20%|██ | 63/311 [00:00<00:01, 175.84it/s, Materializing param=model.layers.5.self_attn.k_norm.weight] Loading weights: 20%|██ | 63/311 [00:00<00:01, 175.84it/s, Materializing param=model.layers.5.self_attn.k_norm.weight] Loading weights: 21%|██ | 64/311 [00:00<00:01, 175.84it/s, Materializing param=model.layers.5.self_attn.k_proj.weight] Loading weights: 21%|██ | 64/311 [00:00<00:01, 175.84it/s, Materializing param=model.layers.5.self_attn.k_proj.weight] Loading weights: 21%|██ | 65/311 [00:00<00:01, 175.84it/s, Materializing param=model.layers.5.self_attn.o_proj.weight] Loading weights: 21%|██ | 65/311 [00:00<00:01, 175.84it/s, Materializing param=model.layers.5.self_attn.o_proj.weight] Loading weights: 7%|▋ | 23/311 [00:00<00:24, 11.58it/s, Materializing param=model.layers.1.self_attn.q_proj.weight] Loading weights: 7%|▋ | 23/311 [00:00<00:24, 11.58it/s, Materializing param=model.layers.1.self_attn.q_proj.weight] Loading weights: 21%|██ | 66/311 [00:00<00:01, 175.84it/s, Materializing param=model.layers.5.self_attn.q_norm.weight] Loading weights: 21%|██ | 66/311 [00:00<00:01, 175.84it/s, Materializing param=model.layers.5.self_attn.q_norm.weight] Loading weights: 8%|▊ | 24/311 [00:00<00:24, 11.58it/s, Materializing param=model.layers.1.self_attn.v_proj.weight] Loading weights: 8%|▊ | 24/311 [00:00<00:24, 11.58it/s, Materializing param=model.layers.1.self_attn.v_proj.weight] Loading weights: 22%|██▏ | 67/311 [00:00<00:01, 175.84it/s, Materializing param=model.layers.5.self_attn.q_proj.weight] Loading weights: 22%|██▏ | 67/311 [00:00<00:01, 175.84it/s, Materializing param=model.layers.5.self_attn.q_proj.weight] Loading weights: 17%|█▋ | 54/311 [00:00<00:01, 155.68it/s, Materializing param=model.layers.4.self_attn.o_proj.weight] Loading weights: 17%|█▋ | 54/311 [00:00<00:01, 155.68it/s, Materializing param=model.layers.4.self_attn.o_proj.weight] Loading weights: 18%|█▊ | 55/311 [00:00<00:01, 155.68it/s, Materializing param=model.layers.4.self_attn.q_norm.weight] Loading weights: 18%|█▊ | 55/311 [00:00<00:01, 155.68it/s, Materializing param=model.layers.4.self_attn.q_norm.weight] Loading weights: 8%|▊ | 25/311 [00:00<00:24, 11.58it/s, Materializing param=model.layers.2.input_layernorm.weight] Loading weights: 8%|▊ | 25/311 [00:00<00:24, 11.58it/s, Materializing param=model.layers.2.input_layernorm.weight] Loading weights: 8%|▊ | 26/311 [00:00<00:24, 11.58it/s, Materializing param=model.layers.2.mlp.down_proj.weight] Loading weights: 22%|██▏ | 68/311 [00:00<00:01, 175.84it/s, Materializing param=model.layers.5.self_attn.v_proj.weight] Loading weights: 8%|▊ | 26/311 [00:00<00:24, 11.58it/s, Materializing param=model.layers.2.mlp.down_proj.weight] Loading weights: 22%|██▏ | 68/311 [00:00<00:01, 175.84it/s, Materializing param=model.layers.5.self_attn.v_proj.weight] Loading weights: 22%|██▏ | 69/311 [00:00<00:01, 175.84it/s, Materializing param=model.layers.6.input_layernorm.weight] Loading weights: 22%|██▏ | 69/311 [00:00<00:01, 175.84it/s, Materializing param=model.layers.6.input_layernorm.weight] Loading weights: 23%|██▎ | 72/311 [00:00<00:01, 158.89it/s, Materializing param=model.layers.6.mlp.up_proj.weight] Loading weights: 23%|██▎ | 70/311 [00:00<00:01, 175.84it/s, Materializing param=model.layers.6.mlp.down_proj.weight] Loading weights: 23%|██▎ | 72/311 [00:00<00:01, 158.89it/s, Materializing param=model.layers.6.mlp.up_proj.weight] Loading weights: 23%|██▎ | 70/311 [00:00<00:01, 175.84it/s, Materializing param=model.layers.6.mlp.down_proj.weight] Loading weights: 18%|█▊ | 56/311 [00:00<00:01, 155.68it/s, Materializing param=model.layers.4.self_attn.q_proj.weight] Loading weights: 18%|█▊ | 56/311 [00:00<00:01, 155.68it/s, Materializing param=model.layers.4.self_attn.q_proj.weight] Loading weights: 18%|█▊ | 57/311 [00:00<00:01, 155.68it/s, Materializing param=model.layers.4.self_attn.v_proj.weight] Loading weights: 18%|█▊ | 57/311 [00:00<00:01, 155.68it/s, Materializing param=model.layers.4.self_attn.v_proj.weight] Loading weights: 13%|█▎ | 40/311 [00:00<00:02, 114.67it/s, Materializing param=model.layers.3.post_attention_layernorm.weight] Loading weights: 13%|█▎ | 40/311 [00:00<00:02, 114.67it/s, Materializing param=model.layers.3.post_attention_layernorm.weight] Loading weights: 13%|█▎ | 41/311 [00:00<00:02, 114.67it/s, Materializing param=model.layers.3.self_attn.k_norm.weight] Loading weights: 13%|█▎ | 41/311 [00:00<00:02, 114.67it/s, Materializing param=model.layers.3.self_attn.k_norm.weight] Loading weights: 14%|█▎ | 42/311 [00:00<00:02, 114.67it/s, Materializing param=model.layers.3.self_attn.k_proj.weight] Loading weights: 14%|█▎ | 42/311 [00:00<00:02, 114.67it/s, Materializing param=model.layers.3.self_attn.k_proj.weight] Loading weights: 20%|█▉ | 62/311 [00:00<00:01, 132.29it/s, Materializing param=model.layers.5.post_attention_layernorm.weight] Loading weights: 20%|█▉ | 62/311 [00:00<00:01, 132.29it/s, Materializing param=model.layers.5.post_attention_layernorm.weight] Loading weights: 19%|█▊ | 58/311 [00:00<00:01, 155.68it/s, Materializing param=model.layers.5.input_layernorm.weight] Loading weights: 19%|█▊ | 58/311 [00:00<00:01, 155.68it/s, Materializing param=model.layers.5.input_layernorm.weight] Loading weights: 19%|█▉ | 59/311 [00:00<00:01, 155.68it/s, Materializing param=model.layers.5.mlp.down_proj.weight] Loading weights: 19%|█▉ | 59/311 [00:00<00:01, 155.68it/s, Materializing param=model.layers.5.mlp.down_proj.weight] Loading weights: 20%|██ | 63/311 [00:00<00:01, 132.29it/s, Materializing param=model.layers.5.self_attn.k_norm.weight] Loading weights: 14%|█▍ | 43/311 [00:00<00:02, 114.67it/s, Materializing param=model.layers.3.self_attn.o_proj.weight] Loading weights: 14%|█▍ | 43/311 [00:00<00:02, 114.67it/s, Materializing param=model.layers.3.self_attn.o_proj.weight] Loading weights: 20%|██ | 63/311 [00:00<00:01, 132.29it/s, Materializing param=model.layers.5.self_attn.k_norm.weight] Loading weights: 21%|██ | 64/311 [00:00<00:01, 132.29it/s, Materializing param=model.layers.5.self_attn.k_proj.weight] Loading weights: 21%|██ | 64/311 [00:00<00:01, 132.29it/s, Materializing param=model.layers.5.self_attn.k_proj.weight] Loading weights: 21%|██ | 65/311 [00:00<00:01, 132.29it/s, Materializing param=model.layers.5.self_attn.o_proj.weight] Loading weights: 21%|██ | 65/311 [00:00<00:01, 132.29it/s, Materializing param=model.layers.5.self_attn.o_proj.weight] Loading weights: 14%|█▍ | 44/311 [00:00<00:02, 114.67it/s, Materializing param=model.layers.3.self_attn.q_norm.weight] Loading weights: 14%|█▍ | 44/311 [00:00<00:02, 114.67it/s, Materializing param=model.layers.3.self_attn.q_norm.weight] Loading weights: 14%|█▍ | 45/311 [00:00<00:02, 114.67it/s, Materializing param=model.layers.3.self_attn.q_proj.weight] Loading weights: 14%|█▍ | 45/311 [00:00<00:02, 114.67it/s, Materializing param=model.layers.3.self_attn.q_proj.weight] Loading weights: 23%|██▎ | 73/311 [00:00<00:01, 158.89it/s, Materializing param=model.layers.6.post_attention_layernorm.weight] Loading weights: 23%|██▎ | 73/311 [00:00<00:01, 158.89it/s, Materializing param=model.layers.6.post_attention_layernorm.weight] Loading weights: 24%|██▍ | 74/311 [00:00<00:01, 158.89it/s, Materializing param=model.layers.6.self_attn.k_norm.weight] Loading weights: 24%|██▍ | 74/311 [00:00<00:01, 158.89it/s, Materializing param=model.layers.6.self_attn.k_norm.weight] Loading weights: 23%|██▎ | 71/311 [00:00<00:01, 175.84it/s, Materializing param=model.layers.6.mlp.gate_proj.weight] Loading weights: 21%|██ | 66/311 [00:00<00:01, 132.29it/s, Materializing param=model.layers.5.self_attn.q_norm.weight] Loading weights: 23%|██▎ | 71/311 [00:00<00:01, 175.84it/s, Materializing param=model.layers.6.mlp.gate_proj.weight] Loading weights: 21%|██ | 66/311 [00:00<00:01, 132.29it/s, Materializing param=model.layers.5.self_attn.q_norm.weight] Loading weights: 24%|██▍ | 75/311 [00:00<00:01, 158.89it/s, Materializing param=model.layers.6.self_attn.k_proj.weight] Loading weights: 24%|██▍ | 75/311 [00:00<00:01, 158.89it/s, Materializing param=model.layers.6.self_attn.k_proj.weight] Loading weights: 22%|██▏ | 67/311 [00:00<00:01, 132.29it/s, Materializing param=model.layers.5.self_attn.q_proj.weight] Loading weights: 22%|██▏ | 67/311 [00:00<00:01, 132.29it/s, Materializing param=model.layers.5.self_attn.q_proj.weight] Loading weights: 24%|██▍ | 76/311 [00:00<00:01, 158.89it/s, Materializing param=model.layers.6.self_attn.o_proj.weight] Loading weights: 24%|██▍ | 76/311 [00:00<00:01, 158.89it/s, Materializing param=model.layers.6.self_attn.o_proj.weight] Loading weights: 9%|▊ | 27/311 [00:00<00:24, 11.58it/s, Materializing param=model.layers.2.mlp.gate_proj.weight] Loading weights: 9%|▊ | 27/311 [00:00<00:24, 11.58it/s, Materializing param=model.layers.2.mlp.gate_proj.weight] Loading weights: 15%|█▍ | 46/311 [00:00<00:02, 114.67it/s, Materializing param=model.layers.3.self_attn.v_proj.weight] Loading weights: 15%|█▍ | 46/311 [00:00<00:02, 114.67it/s, Materializing param=model.layers.3.self_attn.v_proj.weight] Loading weights: 22%|██▏ | 68/311 [00:00<00:01, 132.29it/s, Materializing param=model.layers.5.self_attn.v_proj.weight] Loading weights: 19%|█▉ | 60/311 [00:00<00:01, 155.68it/s, Materializing param=model.layers.5.mlp.gate_proj.weight] Loading weights: 22%|██▏ | 68/311 [00:00<00:01, 132.29it/s, Materializing param=model.layers.5.self_attn.v_proj.weight] Loading weights: 19%|█▉ | 60/311 [00:00<00:01, 155.68it/s, Materializing param=model.layers.5.mlp.gate_proj.weight] Loading weights: 25%|██▍ | 77/311 [00:00<00:01, 158.89it/s, Materializing param=model.layers.6.self_attn.q_norm.weight] Loading weights: 25%|██▍ | 77/311 [00:00<00:01, 158.89it/s, Materializing param=model.layers.6.self_attn.q_norm.weight] Loading weights: 15%|█▌ | 47/311 [00:00<00:02, 114.67it/s, Materializing param=model.layers.4.input_layernorm.weight] Loading weights: 15%|█▌ | 47/311 [00:00<00:02, 114.67it/s, Materializing param=model.layers.4.input_layernorm.weight] Loading weights: 15%|█▌ | 48/311 [00:00<00:02, 114.67it/s, Materializing param=model.layers.4.mlp.down_proj.weight] Loading weights: 15%|█▌ | 48/311 [00:00<00:02, 114.67it/s, Materializing param=model.layers.4.mlp.down_proj.weight] Loading weights: 22%|██▏ | 69/311 [00:00<00:01, 132.29it/s, Materializing param=model.layers.6.input_layernorm.weight] Loading weights: 22%|██▏ | 69/311 [00:00<00:01, 132.29it/s, Materializing param=model.layers.6.input_layernorm.weight] Loading weights: 23%|██▎ | 70/311 [00:00<00:01, 132.29it/s, Materializing param=model.layers.6.mlp.down_proj.weight] Loading weights: 23%|██▎ | 70/311 [00:00<00:01, 132.29it/s, Materializing param=model.layers.6.mlp.down_proj.weight] Loading weights: 25%|██▌ | 78/311 [00:00<00:01, 158.89it/s, Materializing param=model.layers.6.self_attn.q_proj.weight] Loading weights: 25%|██▌ | 78/311 [00:00<00:01, 158.89it/s, Materializing param=model.layers.6.self_attn.q_proj.weight] Loading weights: 25%|██▌ | 79/311 [00:00<00:01, 158.89it/s, Materializing param=model.layers.6.self_attn.v_proj.weight] Loading weights: 25%|██▌ | 79/311 [00:00<00:01, 158.89it/s, Materializing param=model.layers.6.self_attn.v_proj.weight] Loading weights: 23%|██▎ | 72/311 [00:00<00:01, 175.84it/s, Materializing param=model.layers.6.mlp.up_proj.weight] Loading weights: 23%|██▎ | 72/311 [00:00<00:01, 175.84it/s, Materializing param=model.layers.6.mlp.up_proj.weight] Loading weights: 26%|██▌ | 80/311 [00:00<00:01, 158.89it/s, Materializing param=model.layers.7.input_layernorm.weight] Loading weights: 26%|██▌ | 80/311 [00:00<00:01, 158.89it/s, Materializing param=model.layers.7.input_layernorm.weight] Loading weights: 26%|██▌ | 81/311 [00:00<00:01, 158.89it/s, Materializing param=model.layers.7.mlp.down_proj.weight] Loading weights: 26%|██▌ | 81/311 [00:00<00:01, 158.89it/s, Materializing param=model.layers.7.mlp.down_proj.weight] Loading weights: 20%|█▉ | 61/311 [00:00<00:01, 155.68it/s, Materializing param=model.layers.5.mlp.up_proj.weight] Loading weights: 20%|█▉ | 61/311 [00:00<00:01, 155.68it/s, Materializing param=model.layers.5.mlp.up_proj.weight] Loading weights: 9%|▉ | 28/311 [00:00<00:24, 11.58it/s, Materializing param=model.layers.2.mlp.up_proj.weight] Loading weights: 9%|▉ | 28/311 [00:00<00:24, 11.58it/s, Materializing param=model.layers.2.mlp.up_proj.weight] Loading weights: 23%|██▎ | 73/311 [00:00<00:01, 175.84it/s, Materializing param=model.layers.6.post_attention_layernorm.weight] Loading weights: 23%|██▎ | 71/311 [00:00<00:01, 132.29it/s, Materializing param=model.layers.6.mlp.gate_proj.weight] Loading weights: 23%|██▎ | 73/311 [00:00<00:01, 175.84it/s, Materializing param=model.layers.6.post_attention_layernorm.weight] Loading weights: 23%|██▎ | 71/311 [00:00<00:01, 132.29it/s, Materializing param=model.layers.6.mlp.gate_proj.weight] Loading weights: 24%|██▍ | 74/311 [00:00<00:01, 175.84it/s, Materializing param=model.layers.6.self_attn.k_norm.weight] Loading weights: 24%|██▍ | 74/311 [00:00<00:01, 175.84it/s, Materializing param=model.layers.6.self_attn.k_norm.weight] Loading weights: 24%|██▍ | 75/311 [00:00<00:01, 175.84it/s, Materializing param=model.layers.6.self_attn.k_proj.weight] Loading weights: 24%|██▍ | 75/311 [00:00<00:01, 175.84it/s, Materializing param=model.layers.6.self_attn.k_proj.weight] Loading weights: 16%|█▌ | 49/311 [00:00<00:02, 114.67it/s, Materializing param=model.layers.4.mlp.gate_proj.weight] Loading weights: 16%|█▌ | 49/311 [00:00<00:02, 114.67it/s, Materializing param=model.layers.4.mlp.gate_proj.weight] Loading weights: 24%|██▍ | 76/311 [00:00<00:01, 175.84it/s, Materializing param=model.layers.6.self_attn.o_proj.weight] Loading weights: 24%|██▍ | 76/311 [00:00<00:01, 175.84it/s, Materializing param=model.layers.6.self_attn.o_proj.weight] Loading weights: 25%|██▍ | 77/311 [00:00<00:01, 175.84it/s, Materializing param=model.layers.6.self_attn.q_norm.weight] Loading weights: 25%|██▍ | 77/311 [00:00<00:01, 175.84it/s, Materializing param=model.layers.6.self_attn.q_norm.weight] Loading weights: 25%|██▌ | 78/311 [00:00<00:01, 175.84it/s, Materializing param=model.layers.6.self_attn.q_proj.weight] Loading weights: 25%|██▌ | 78/311 [00:00<00:01, 175.84it/s, Materializing param=model.layers.6.self_attn.q_proj.weight] Loading weights: 20%|█▉ | 62/311 [00:00<00:01, 155.68it/s, Materializing param=model.layers.5.post_attention_layernorm.weight] Loading weights: 20%|█▉ | 62/311 [00:00<00:01, 155.68it/s, Materializing param=model.layers.5.post_attention_layernorm.weight] Loading weights: 20%|██ | 63/311 [00:00<00:01, 155.68it/s, Materializing param=model.layers.5.self_attn.k_norm.weight] Loading weights: 20%|██ | 63/311 [00:00<00:01, 155.68it/s, Materializing param=model.layers.5.self_attn.k_norm.weight] Loading weights: 21%|██ | 64/311 [00:00<00:01, 155.68it/s, Materializing param=model.layers.5.self_attn.k_proj.weight] Loading weights: 21%|██ | 64/311 [00:00<00:01, 155.68it/s, Materializing param=model.layers.5.self_attn.k_proj.weight] Loading weights: 21%|██ | 65/311 [00:00<00:01, 155.68it/s, Materializing param=model.layers.5.self_attn.o_proj.weight] Loading weights: 21%|██ | 65/311 [00:00<00:01, 155.68it/s, Materializing param=model.layers.5.self_attn.o_proj.weight] Loading weights: 25%|██▌ | 79/311 [00:00<00:01, 175.84it/s, Materializing param=model.layers.6.self_attn.v_proj.weight] Loading weights: 25%|██▌ | 79/311 [00:00<00:01, 175.84it/s, Materializing param=model.layers.6.self_attn.v_proj.weight] Loading weights: 9%|▉ | 29/311 [00:00<00:24, 11.58it/s, Materializing param=model.layers.2.post_attention_layernorm.weight] Loading weights: 26%|██▌ | 80/311 [00:00<00:01, 175.84it/s, Materializing param=model.layers.7.input_layernorm.weight] Loading weights: 1%| | 3/311 [00:00<00:24, 12.67it/s, Materializing param=model.layers.0.input_layernorm.weight] Loading weights: 9%|▉ | 29/311 [00:00<00:24, 11.58it/s, Materializing param=model.layers.2.post_attention_layernorm.weight] Loading weights: 26%|██▌ | 80/311 [00:00<00:01, 175.84it/s, Materializing param=model.layers.7.input_layernorm.weight] Loading weights: 1%| | 3/311 [00:00<00:24, 12.67it/s, Materializing param=model.layers.0.input_layernorm.weight] Loading weights: 10%|▉ | 30/311 [00:00<00:24, 11.58it/s, Materializing param=model.layers.2.self_attn.k_norm.weight] Loading weights: 26%|██▌ | 81/311 [00:00<00:01, 175.84it/s, Materializing param=model.layers.7.mlp.down_proj.weight] Loading weights: 10%|▉ | 30/311 [00:00<00:24, 11.58it/s, Materializing param=model.layers.2.self_attn.k_norm.weight] Loading weights: 26%|██▌ | 81/311 [00:00<00:01, 175.84it/s, Materializing param=model.layers.7.mlp.down_proj.weight] Loading weights: 1%|▏ | 4/311 [00:00<00:26, 11.53it/s, Materializing param=model.layers.0.input_layernorm.weight] Loading weights: 23%|██▎ | 72/311 [00:00<00:01, 132.29it/s, Materializing param=model.layers.6.mlp.up_proj.weight] Loading weights: 10%|▉ | 31/311 [00:00<00:24, 11.58it/s, Materializing param=model.layers.2.self_attn.k_proj.weight] Loading weights: 1%|▏ | 4/311 [00:00<00:26, 11.53it/s, Materializing param=model.layers.0.mlp.down_proj.weight] Loading weights: 23%|██▎ | 72/311 [00:00<00:01, 132.29it/s, Materializing param=model.layers.6.mlp.up_proj.weight] Loading weights: 10%|▉ | 31/311 [00:00<00:24, 11.58it/s, Materializing param=model.layers.2.self_attn.k_proj.weight] Loading weights: 1%|▏ | 4/311 [00:00<00:26, 11.53it/s, Materializing param=model.layers.0.mlp.down_proj.weight] Loading weights: 21%|██ | 66/311 [00:00<00:01, 155.68it/s, Materializing param=model.layers.5.self_attn.q_norm.weight] Loading weights: 21%|██ | 66/311 [00:00<00:01, 155.68it/s, Materializing param=model.layers.5.self_attn.q_norm.weight] Loading weights: 26%|██▋ | 82/311 [00:00<00:01, 158.89it/s, Materializing param=model.layers.7.mlp.gate_proj.weight] Loading weights: 26%|██▋ | 82/311 [00:00<00:01, 158.89it/s, Materializing param=model.layers.7.mlp.gate_proj.weight] Loading weights: 10%|█ | 32/311 [00:00<00:24, 11.58it/s, Materializing param=model.layers.2.self_attn.o_proj.weight] Loading weights: 10%|█ | 32/311 [00:00<00:24, 11.58it/s, Materializing param=model.layers.2.self_attn.o_proj.weight] Loading weights: 27%|██▋ | 83/311 [00:00<00:01, 158.89it/s, Materializing param=model.layers.7.mlp.up_proj.weight] Loading weights: 27%|██▋ | 83/311 [00:00<00:01, 158.89it/s, Materializing param=model.layers.7.mlp.up_proj.weight] Loading weights: 16%|█▌ | 50/311 [00:00<00:02, 114.67it/s, Materializing param=model.layers.4.mlp.up_proj.weight] Loading weights: 16%|█▌ | 50/311 [00:00<00:02, 114.67it/s, Materializing param=model.layers.4.mlp.up_proj.weight] Loading weights: 11%|█ | 33/311 [00:00<00:24, 11.58it/s, Materializing param=model.layers.2.self_attn.q_norm.weight] Loading weights: 11%|█ | 33/311 [00:00<00:24, 11.58it/s, Materializing param=model.layers.2.self_attn.q_norm.weight] Loading weights: 11%|█ | 34/311 [00:00<00:23, 11.58it/s, Materializing param=model.layers.2.self_attn.q_proj.weight] Loading weights: 22%|██▏ | 67/311 [00:00<00:01, 155.68it/s, Materializing param=model.layers.5.self_attn.q_proj.weight] Loading weights: 11%|█ | 34/311 [00:00<00:23, 11.58it/s, Materializing param=model.layers.2.self_attn.q_proj.weight] Loading weights: 22%|██▏ | 67/311 [00:00<00:01, 155.68it/s, Materializing param=model.layers.5.self_attn.q_proj.weight] Loading weights: 22%|██▏ | 68/311 [00:00<00:01, 155.68it/s, Materializing param=model.layers.5.self_attn.v_proj.weight] Loading weights: 22%|██▏ | 68/311 [00:00<00:01, 155.68it/s, Materializing param=model.layers.5.self_attn.v_proj.weight] Loading weights: 22%|██▏ | 69/311 [00:00<00:01, 155.68it/s, Materializing param=model.layers.6.input_layernorm.weight] Loading weights: 22%|██▏ | 69/311 [00:00<00:01, 155.68it/s, Materializing param=model.layers.6.input_layernorm.weight] Loading weights: 23%|██▎ | 70/311 [00:00<00:01, 155.68it/s, Materializing param=model.layers.6.mlp.down_proj.weight] Loading weights: 23%|██▎ | 70/311 [00:00<00:01, 155.68it/s, Materializing param=model.layers.6.mlp.down_proj.weight] Loading weights: 11%|█▏ | 35/311 [00:00<00:23, 11.58it/s, Materializing param=model.layers.2.self_attn.v_proj.weight] Loading weights: 26%|██▋ | 82/311 [00:00<00:01, 175.84it/s, Materializing param=model.layers.7.mlp.gate_proj.weight] Loading weights: 11%|█▏ | 35/311 [00:00<00:23, 11.58it/s, Materializing param=model.layers.2.self_attn.v_proj.weight] Loading weights: 26%|██▋ | 82/311 [00:00<00:01, 175.84it/s, Materializing param=model.layers.7.mlp.gate_proj.weight] Loading weights: 2%|▏ | 5/311 [00:00<00:26, 11.53it/s, Materializing param=model.layers.0.mlp.gate_proj.weight] Loading weights: 23%|██▎ | 73/311 [00:00<00:01, 132.29it/s, Materializing param=model.layers.6.post_attention_layernorm.weight] Loading weights: 2%|▏ | 5/311 [00:00<00:26, 11.53it/s, Materializing param=model.layers.0.mlp.gate_proj.weight] Loading weights: 23%|██▎ | 73/311 [00:00<00:01, 132.29it/s, Materializing param=model.layers.6.post_attention_layernorm.weight] Loading weights: 24%|██▍ | 74/311 [00:00<00:01, 132.29it/s, Materializing param=model.layers.6.self_attn.k_norm.weight] Loading weights: 24%|██▍ | 74/311 [00:00<00:01, 132.29it/s, Materializing param=model.layers.6.self_attn.k_norm.weight] Loading weights: 24%|██▍ | 75/311 [00:00<00:01, 132.29it/s, Materializing param=model.layers.6.self_attn.k_proj.weight] Loading weights: 24%|██▍ | 75/311 [00:00<00:01, 132.29it/s, Materializing param=model.layers.6.self_attn.k_proj.weight] Loading weights: 12%|█▏ | 36/311 [00:00<00:23, 11.58it/s, Materializing param=model.layers.3.input_layernorm.weight] Loading weights: 27%|██▋ | 84/311 [00:00<00:01, 158.89it/s, Materializing param=model.layers.7.post_attention_layernorm.weight] Loading weights: 12%|█▏ | 36/311 [00:00<00:23, 11.58it/s, Materializing param=model.layers.3.input_layernorm.weight] Loading weights: 27%|██▋ | 84/311 [00:00<00:01, 158.89it/s, Materializing param=model.layers.7.post_attention_layernorm.weight] Loading weights: 12%|█▏ | 37/311 [00:00<00:23, 11.58it/s, Materializing param=model.layers.3.mlp.down_proj.weight] Loading weights: 24%|██▍ | 76/311 [00:00<00:01, 132.29it/s, Materializing param=model.layers.6.self_attn.o_proj.weight] Loading weights: 12%|█▏ | 37/311 [00:00<00:23, 11.58it/s, Materializing param=model.layers.3.mlp.down_proj.weight] Loading weights: 24%|██▍ | 76/311 [00:00<00:01, 132.29it/s, Materializing param=model.layers.6.self_attn.o_proj.weight] Loading weights: 27%|██▋ | 85/311 [00:00<00:01, 158.89it/s, Materializing param=model.layers.7.self_attn.k_norm.weight] Loading weights: 27%|██▋ | 85/311 [00:00<00:01, 158.89it/s, Materializing param=model.layers.7.self_attn.k_norm.weight] Loading weights: 28%|██▊ | 86/311 [00:00<00:01, 158.89it/s, Materializing param=model.layers.7.self_attn.k_proj.weight] Loading weights: 28%|██▊ | 86/311 [00:00<00:01, 158.89it/s, Materializing param=model.layers.7.self_attn.k_proj.weight] Loading weights: 28%|██▊ | 87/311 [00:00<00:01, 158.89it/s, Materializing param=model.layers.7.self_attn.o_proj.weight] Loading weights: 28%|██▊ | 87/311 [00:00<00:01, 158.89it/s, Materializing param=model.layers.7.self_attn.o_proj.weight] Loading weights: 23%|██▎ | 71/311 [00:00<00:01, 155.68it/s, Materializing param=model.layers.6.mlp.gate_proj.weight] Loading weights: 23%|██▎ | 71/311 [00:00<00:01, 155.68it/s, Materializing param=model.layers.6.mlp.gate_proj.weight] Loading weights: 16%|█▋ | 51/311 [00:00<00:02, 114.67it/s, Materializing param=model.layers.4.post_attention_layernorm.weight] Loading weights: 16%|█▋ | 51/311 [00:00<00:02, 114.67it/s, Materializing param=model.layers.4.post_attention_layernorm.weight] Loading weights: 17%|█▋ | 52/311 [00:00<00:02, 114.67it/s, Materializing param=model.layers.4.self_attn.k_norm.weight] Loading weights: 17%|█▋ | 52/311 [00:00<00:02, 114.67it/s, Materializing param=model.layers.4.self_attn.k_norm.weight] Loading weights: 17%|█▋ | 53/311 [00:00<00:02, 114.67it/s, Materializing param=model.layers.4.self_attn.k_proj.weight] Loading weights: 17%|█▋ | 53/311 [00:00<00:02, 114.67it/s, Materializing param=model.layers.4.self_attn.k_proj.weight] Loading weights: 27%|██▋ | 83/311 [00:00<00:01, 175.84it/s, Materializing param=model.layers.7.mlp.up_proj.weight] Loading weights: 27%|██▋ | 83/311 [00:00<00:01, 175.84it/s, Materializing param=model.layers.7.mlp.up_proj.weight] Loading weights: 17%|█▋ | 54/311 [00:00<00:02, 114.67it/s, Materializing param=model.layers.4.self_attn.o_proj.weight] Loading weights: 17%|█▋ | 54/311 [00:00<00:02, 114.67it/s, Materializing param=model.layers.4.self_attn.o_proj.weight] Loading weights: 25%|██▍ | 77/311 [00:00<00:01, 132.29it/s, Materializing param=model.layers.6.self_attn.q_norm.weight] Loading weights: 25%|██▍ | 77/311 [00:00<00:01, 132.29it/s, Materializing param=model.layers.6.self_attn.q_norm.weight] Loading weights: 25%|██▌ | 78/311 [00:00<00:01, 132.29it/s, Materializing param=model.layers.6.self_attn.q_proj.weight] Loading weights: 28%|██▊ | 88/311 [00:00<00:01, 158.89it/s, Materializing param=model.layers.7.self_attn.q_norm.weight] Loading weights: 28%|██▊ | 88/311 [00:00<00:01, 158.89it/s, Materializing param=model.layers.7.self_attn.q_norm.weight] Loading weights: 25%|██▌ | 78/311 [00:00<00:01, 132.29it/s, Materializing param=model.layers.6.self_attn.q_proj.weight] Loading weights: 25%|██▌ | 79/311 [00:00<00:01, 132.29it/s, Materializing param=model.layers.6.self_attn.v_proj.weight] Loading weights: 25%|██▌ | 79/311 [00:00<00:01, 132.29it/s, Materializing param=model.layers.6.self_attn.v_proj.weight] Loading weights: 29%|██▊ | 89/311 [00:00<00:01, 158.89it/s, Materializing param=model.layers.7.self_attn.q_proj.weight] Loading weights: 29%|██▊ | 89/311 [00:00<00:01, 158.89it/s, Materializing param=model.layers.7.self_attn.q_proj.weight] Loading weights: 26%|██▌ | 80/311 [00:00<00:01, 132.29it/s, Materializing param=model.layers.7.input_layernorm.weight] Loading weights: 26%|██▌ | 80/311 [00:00<00:01, 132.29it/s, Materializing param=model.layers.7.input_layernorm.weight] Loading weights: 26%|██▌ | 81/311 [00:00<00:01, 132.29it/s, Materializing param=model.layers.7.mlp.down_proj.weight] Loading weights: 2%|▏ | 6/311 [00:00<00:26, 11.53it/s, Materializing param=model.layers.0.mlp.up_proj.weight] Loading weights: 26%|██▌ | 81/311 [00:00<00:01, 132.29it/s, Materializing param=model.layers.7.mlp.down_proj.weight] Loading weights: 2%|▏ | 6/311 [00:00<00:26, 11.53it/s, Materializing param=model.layers.0.mlp.up_proj.weight] Loading weights: 29%|██▉ | 90/311 [00:00<00:01, 158.89it/s, Materializing param=model.layers.7.self_attn.v_proj.weight] Loading weights: 29%|██▉ | 90/311 [00:00<00:01, 158.89it/s, Materializing param=model.layers.7.self_attn.v_proj.weight] Loading weights: 18%|█▊ | 55/311 [00:00<00:02, 114.67it/s, Materializing param=model.layers.4.self_attn.q_norm.weight] Loading weights: 12%|█▏ | 38/311 [00:00<00:23, 11.58it/s, Materializing param=model.layers.3.mlp.gate_proj.weight] Loading weights: 18%|█▊ | 55/311 [00:00<00:02, 114.67it/s, Materializing param=model.layers.4.self_attn.q_norm.weight] Loading weights: 12%|█▏ | 38/311 [00:00<00:23, 11.58it/s, Materializing param=model.layers.3.mlp.gate_proj.weight] Loading weights: 18%|█▊ | 56/311 [00:00<00:02, 114.67it/s, Materializing param=model.layers.4.self_attn.q_proj.weight] Loading weights: 18%|█▊ | 56/311 [00:00<00:02, 114.67it/s, Materializing param=model.layers.4.self_attn.q_proj.weight] Loading weights: 29%|██▉ | 91/311 [00:00<00:01, 158.89it/s, Materializing param=model.layers.8.input_layernorm.weight] Loading weights: 29%|██▉ | 91/311 [00:00<00:01, 158.89it/s, Materializing param=model.layers.8.input_layernorm.weight] Loading weights: 30%|██▉ | 92/311 [00:00<00:01, 158.89it/s, Materializing param=model.layers.8.mlp.down_proj.weight] Loading weights: 23%|██▎ | 72/311 [00:00<00:01, 155.68it/s, Materializing param=model.layers.6.mlp.up_proj.weight] Loading weights: 30%|██▉ | 92/311 [00:00<00:01, 158.89it/s, Materializing param=model.layers.8.mlp.down_proj.weight] Loading weights: 23%|██▎ | 72/311 [00:00<00:01, 155.68it/s, Materializing param=model.layers.6.mlp.up_proj.weight][INFO|dynamic_module_utils.py:406] 2026-04-29 17:06:08,005 >> Could not locate the custom_generate/generate.py inside Qwen/Qwen3-1.7B. + Loading weights: 27%|██▋ | 84/311 [00:00<00:01, 175.84it/s, Materializing param=model.layers.7.post_attention_layernorm.weight] Loading weights: 27%|██▋ | 84/311 [00:00<00:01, 175.84it/s, Materializing param=model.layers.7.post_attention_layernorm.weight] Loading weights: 18%|█▊ | 57/311 [00:00<00:02, 114.67it/s, Materializing param=model.layers.4.self_attn.v_proj.weight] Loading weights: 18%|█▊ | 57/311 [00:00<00:02, 114.67it/s, Materializing param=model.layers.4.self_attn.v_proj.weight] Loading weights: 27%|██▋ | 85/311 [00:00<00:01, 175.84it/s, Materializing param=model.layers.7.self_attn.k_norm.weight] Loading weights: 27%|██▋ | 85/311 [00:00<00:01, 175.84it/s, Materializing param=model.layers.7.self_attn.k_norm.weight] Loading weights: 28%|██▊ | 86/311 [00:00<00:01, 175.84it/s, Materializing param=model.layers.7.self_attn.k_proj.weight] Loading weights: 28%|██▊ | 86/311 [00:00<00:01, 175.84it/s, Materializing param=model.layers.7.self_attn.k_proj.weight] Loading weights: 28%|██▊ | 87/311 [00:00<00:01, 175.84it/s, Materializing param=model.layers.7.self_attn.o_proj.weight] Loading weights: 28%|██▊ | 87/311 [00:00<00:01, 175.84it/s, Materializing param=model.layers.7.self_attn.o_proj.weight][INFO|2026-04-29 17:06:08] llamafactory.model.model_utils.checkpointing:144 >> Gradient checkpointing enabled. +[INFO|2026-04-29 17:06:08] llamafactory.model.model_utils.attention:144 >> Using torch SDPA for faster training and inference. +[INFO|2026-04-29 17:06:08] llamafactory.model.adapter:144 >> Upcasting trainable params to float32. +[INFO|2026-04-29 17:06:08] llamafactory.model.adapter:144 >> Fine-tuning method: Full + Loading weights: 19%|█▊ | 58/311 [00:00<00:02, 114.67it/s, Materializing param=model.layers.5.input_layernorm.weight] Loading weights: 19%|█▊ | 58/311 [00:00<00:02, 114.67it/s, Materializing param=model.layers.5.input_layernorm.weight] Loading weights: 19%|█▉ | 59/311 [00:00<00:02, 114.67it/s, Materializing param=model.layers.5.mlp.down_proj.weight] Loading weights: 19%|█▉ | 59/311 [00:00<00:02, 114.67it/s, Materializing param=model.layers.5.mlp.down_proj.weight] Loading weights: 26%|██▋ | 82/311 [00:00<00:01, 132.29it/s, Materializing param=model.layers.7.mlp.gate_proj.weight] Loading weights: 26%|██▋ | 82/311 [00:00<00:01, 132.29it/s, Materializing param=model.layers.7.mlp.gate_proj.weight] Loading weights: 28%|██▊ | 88/311 [00:00<00:01, 175.84it/s, Materializing param=model.layers.7.self_attn.q_norm.weight] Loading weights: 28%|██▊ | 88/311 [00:00<00:01, 175.84it/s, Materializing param=model.layers.7.self_attn.q_norm.weight] Loading weights: 29%|██▊ | 89/311 [00:00<00:01, 175.84it/s, Materializing param=model.layers.7.self_attn.q_proj.weight] Loading weights: 29%|██▊ | 89/311 [00:00<00:01, 175.84it/s, Materializing param=model.layers.7.self_attn.q_proj.weight] Loading weights: 30%|██▉ | 93/311 [00:00<00:00, 254.36it/s, Materializing param=model.layers.8.mlp.down_proj.weight] Loading weights: 30%|██▉ | 93/311 [00:00<00:00, 254.36it/s, Materializing param=model.layers.8.mlp.gate_proj.weight] Loading weights: 23%|██▎ | 73/311 [00:00<00:01, 155.68it/s, Materializing param=model.layers.6.post_attention_layernorm.weight] Loading weights: 30%|██▉ | 93/311 [00:00<00:00, 254.36it/s, Materializing param=model.layers.8.mlp.gate_proj.weight] Loading weights: 29%|██▉ | 90/311 [00:00<00:01, 175.84it/s, Materializing param=model.layers.7.self_attn.v_proj.weight] Loading weights: 23%|██▎ | 73/311 [00:00<00:01, 155.68it/s, Materializing param=model.layers.6.post_attention_layernorm.weight] Loading weights: 2%|▏ | 7/311 [00:00<00:26, 11.53it/s, Materializing param=model.layers.0.post_attention_layernorm.weight] Loading weights: 29%|██▉ | 90/311 [00:00<00:01, 175.84it/s, Materializing param=model.layers.7.self_attn.v_proj.weight] Loading weights: 2%|▏ | 7/311 [00:00<00:26, 11.53it/s, Materializing param=model.layers.0.post_attention_layernorm.weight] Loading weights: 3%|▎ | 8/311 [00:00<00:26, 11.53it/s, Materializing param=model.layers.0.self_attn.k_norm.weight] Loading weights: 3%|▎ | 8/311 [00:00<00:26, 11.53it/s, Materializing param=model.layers.0.self_attn.k_norm.weight] Loading weights: 3%|▎ | 9/311 [00:00<00:26, 11.53it/s, Materializing param=model.layers.0.self_attn.k_proj.weight] Loading weights: 24%|██▍ | 74/311 [00:00<00:01, 155.68it/s, Materializing param=model.layers.6.self_attn.k_norm.weight] Loading weights: 3%|▎ | 9/311 [00:00<00:26, 11.53it/s, Materializing param=model.layers.0.self_attn.k_proj.weight] Loading weights: 24%|██▍ | 74/311 [00:00<00:01, 155.68it/s, Materializing param=model.layers.6.self_attn.k_norm.weight] Loading weights: 13%|█▎ | 39/311 [00:00<00:02, 120.27it/s, Materializing param=model.layers.3.mlp.gate_proj.weight] Loading weights: 13%|█▎ | 39/311 [00:00<00:02, 120.27it/s, Materializing param=model.layers.3.mlp.up_proj.weight] Loading weights: 13%|█▎ | 39/311 [00:00<00:02, 120.27it/s, Materializing param=model.layers.3.mlp.up_proj.weight] Loading weights: 29%|██▉ | 91/311 [00:00<00:01, 175.84it/s, Materializing param=model.layers.8.input_layernorm.weight] Loading weights: 3%|▎ | 10/311 [00:00<00:26, 11.53it/s, Materializing param=model.layers.0.self_attn.o_proj.weight] Loading weights: 24%|██▍ | 75/311 [00:00<00:01, 155.68it/s, Materializing param=model.layers.6.self_attn.k_proj.weight] Loading weights: 29%|██▉ | 91/311 [00:00<00:01, 175.84it/s, Materializing param=model.layers.8.input_layernorm.weight] Loading weights: 3%|▎ | 10/311 [00:00<00:26, 11.53it/s, Materializing param=model.layers.0.self_attn.o_proj.weight] Loading weights: 24%|██▍ | 75/311 [00:00<00:01, 155.68it/s, Materializing param=model.layers.6.self_attn.k_proj.weight] Loading weights: 30%|██▉ | 92/311 [00:00<00:01, 175.84it/s, Materializing param=model.layers.8.mlp.down_proj.weight] Loading weights: 30%|██▉ | 92/311 [00:00<00:01, 175.84it/s, Materializing param=model.layers.8.mlp.down_proj.weight] Loading weights: 24%|██▍ | 76/311 [00:00<00:01, 155.68it/s, Materializing param=model.layers.6.self_attn.o_proj.weight] Loading weights: 24%|██▍ | 76/311 [00:00<00:01, 155.68it/s, Materializing param=model.layers.6.self_attn.o_proj.weight] Loading weights: 25%|██▍ | 77/311 [00:00<00:01, 155.68it/s, Materializing param=model.layers.6.self_attn.q_norm.weight] Loading weights: 25%|██▍ | 77/311 [00:00<00:01, 155.68it/s, Materializing param=model.layers.6.self_attn.q_norm.weight] Loading weights: 25%|██▌ | 78/311 [00:00<00:01, 155.68it/s, Materializing param=model.layers.6.self_attn.q_proj.weight] Loading weights: 25%|██▌ | 78/311 [00:00<00:01, 155.68it/s, Materializing param=model.layers.6.self_attn.q_proj.weight] Loading weights: 30%|███ | 94/311 [00:00<00:00, 254.36it/s, Materializing param=model.layers.8.mlp.up_proj.weight] Loading weights: 27%|██▋ | 83/311 [00:00<00:01, 132.29it/s, Materializing param=model.layers.7.mlp.up_proj.weight] Loading weights: 30%|███ | 94/311 [00:00<00:00, 254.36it/s, Materializing param=model.layers.8.mlp.up_proj.weight] Loading weights: 27%|██▋ | 83/311 [00:00<00:01, 132.29it/s, Materializing param=model.layers.7.mlp.up_proj.weight] Loading weights: 19%|█▉ | 60/311 [00:00<00:02, 114.67it/s, Materializing param=model.layers.5.mlp.gate_proj.weight] Loading weights: 19%|█▉ | 60/311 [00:00<00:02, 114.67it/s, Materializing param=model.layers.5.mlp.gate_proj.weight] Loading weights: 30%|██▉ | 93/311 [00:00<00:01, 175.84it/s, Materializing param=model.layers.8.mlp.gate_proj.weight] Loading weights: 25%|██▌ | 79/311 [00:00<00:01, 155.68it/s, Materializing param=model.layers.6.self_attn.v_proj.weight] Loading weights: 30%|██▉ | 93/311 [00:00<00:01, 175.84it/s, Materializing param=model.layers.8.mlp.gate_proj.weight] Loading weights: 25%|██▌ | 79/311 [00:00<00:01, 155.68it/s, Materializing param=model.layers.6.self_attn.v_proj.weight] Loading weights: 4%|▎ | 11/311 [00:00<00:26, 11.53it/s, Materializing param=model.layers.0.self_attn.q_norm.weight] Loading weights: 4%|▎ | 11/311 [00:00<00:26, 11.53it/s, Materializing param=model.layers.0.self_attn.q_norm.weight] Loading weights: 4%|▍ | 12/311 [00:00<00:25, 11.53it/s, Materializing param=model.layers.0.self_attn.q_proj.weight] Loading weights: 4%|▍ | 12/311 [00:00<00:25, 11.53it/s, Materializing param=model.layers.0.self_attn.q_proj.weight] Loading weights: 4%|▍ | 13/311 [00:00<00:25, 11.53it/s, Materializing param=model.layers.0.self_attn.v_proj.weight] Loading weights: 4%|▍ | 13/311 [00:00<00:25, 11.53it/s, Materializing param=model.layers.0.self_attn.v_proj.weight] Loading weights: 26%|██▌ | 80/311 [00:00<00:01, 155.68it/s, Materializing param=model.layers.7.input_layernorm.weight] Loading weights: 26%|██▌ | 80/311 [00:00<00:01, 155.68it/s, Materializing param=model.layers.7.input_layernorm.weight] Loading weights: 5%|▍ | 14/311 [00:00<00:25, 11.53it/s, Materializing param=model.layers.1.input_layernorm.weight] Loading weights: 5%|▍ | 14/311 [00:00<00:25, 11.53it/s, Materializing param=model.layers.1.input_layernorm.weight] Loading weights: 5%|▍ | 15/311 [00:00<00:25, 11.53it/s, Materializing param=model.layers.1.mlp.down_proj.weight] Loading weights: 5%|▍ | 15/311 [00:00<00:25, 11.53it/s, Materializing param=model.layers.1.mlp.down_proj.weight] Loading weights: 13%|█▎ | 40/311 [00:00<00:02, 120.27it/s, Materializing param=model.layers.3.post_attention_layernorm.weight] Loading weights: 13%|█▎ | 40/311 [00:00<00:02, 120.27it/s, Materializing param=model.layers.3.post_attention_layernorm.weight] Loading weights: 13%|█▎ | 41/311 [00:00<00:02, 120.27it/s, Materializing param=model.layers.3.self_attn.k_norm.weight] Loading weights: 13%|█▎ | 41/311 [00:00<00:02, 120.27it/s, Materializing param=model.layers.3.self_attn.k_norm.weight] Loading weights: 14%|█▎ | 42/311 [00:00<00:02, 120.27it/s, Materializing param=model.layers.3.self_attn.k_proj.weight] Loading weights: 14%|█▎ | 42/311 [00:00<00:02, 120.27it/s, Materializing param=model.layers.3.self_attn.k_proj.weight] Loading weights: 14%|█▍ | 43/311 [00:00<00:02, 120.27it/s, Materializing param=model.layers.3.self_attn.o_proj.weight] Loading weights: 14%|█▍ | 43/311 [00:00<00:02, 120.27it/s, Materializing param=model.layers.3.self_attn.o_proj.weight] Loading weights: 30%|███ | 94/311 [00:00<00:01, 175.84it/s, Materializing param=model.layers.8.mlp.up_proj.weight] Loading weights: 30%|███ | 94/311 [00:00<00:01, 175.84it/s, Materializing param=model.layers.8.mlp.up_proj.weight] Loading weights: 31%|███ | 95/311 [00:00<00:00, 254.36it/s, Materializing param=model.layers.8.post_attention_layernorm.weight] Loading weights: 31%|███ | 95/311 [00:00<00:00, 254.36it/s, Materializing param=model.layers.8.post_attention_layernorm.weight] Loading weights: 27%|██▋ | 84/311 [00:00<00:00, 228.68it/s, Materializing param=model.layers.7.mlp.up_proj.weight] Loading weights: 27%|██▋ | 84/311 [00:00<00:00, 228.68it/s, Materializing param=model.layers.7.post_attention_layernorm.weight] Loading weights: 31%|███ | 96/311 [00:00<00:00, 254.36it/s, Materializing param=model.layers.8.self_attn.k_norm.weight] Loading weights: 27%|██▋ | 84/311 [00:00<00:00, 228.68it/s, Materializing param=model.layers.7.post_attention_layernorm.weight] Loading weights: 31%|███ | 96/311 [00:00<00:00, 254.36it/s, Materializing param=model.layers.8.self_attn.k_norm.weight] Loading weights: 31%|███ | 97/311 [00:00<00:00, 254.36it/s, Materializing param=model.layers.8.self_attn.k_proj.weight] Loading weights: 31%|███ | 97/311 [00:00<00:00, 254.36it/s, Materializing param=model.layers.8.self_attn.k_proj.weight] Loading weights: 14%|█▍ | 44/311 [00:00<00:02, 120.27it/s, Materializing param=model.layers.3.self_attn.q_norm.weight] Loading weights: 14%|█▍ | 44/311 [00:00<00:02, 120.27it/s, Materializing param=model.layers.3.self_attn.q_norm.weight] Loading weights: 32%|███▏ | 98/311 [00:00<00:00, 254.36it/s, Materializing param=model.layers.8.self_attn.o_proj.weight] Loading weights: 32%|███▏ | 98/311 [00:00<00:00, 254.36it/s, Materializing param=model.layers.8.self_attn.o_proj.weight] Loading weights: 27%|██▋ | 85/311 [00:00<00:00, 228.68it/s, Materializing param=model.layers.7.self_attn.k_norm.weight] Loading weights: 14%|█▍ | 45/311 [00:00<00:02, 120.27it/s, Materializing param=model.layers.3.self_attn.q_proj.weight] Loading weights: 14%|█▍ | 45/311 [00:00<00:02, 120.27it/s, Materializing param=model.layers.3.self_attn.q_proj.weight] Loading weights: 27%|██▋ | 85/311 [00:00<00:00, 228.68it/s, Materializing param=model.layers.7.self_attn.k_norm.weight] Loading weights: 20%|█▉ | 61/311 [00:00<00:02, 114.67it/s, Materializing param=model.layers.5.mlp.up_proj.weight] Loading weights: 28%|██▊ | 86/311 [00:00<00:00, 228.68it/s, Materializing param=model.layers.7.self_attn.k_proj.weight] Loading weights: 28%|██▊ | 86/311 [00:00<00:00, 228.68it/s, Materializing param=model.layers.7.self_attn.k_proj.weight] Loading weights: 20%|█▉ | 61/311 [00:00<00:02, 114.67it/s, Materializing param=model.layers.5.mlp.up_proj.weight] Loading weights: 28%|██▊ | 87/311 [00:00<00:00, 228.68it/s, Materializing param=model.layers.7.self_attn.o_proj.weight] Loading weights: 28%|██▊ | 87/311 [00:00<00:00, 228.68it/s, Materializing param=model.layers.7.self_attn.o_proj.weight] Loading weights: 32%|███▏ | 99/311 [00:00<00:00, 254.36it/s, Materializing param=model.layers.8.self_attn.q_norm.weight] Loading weights: 32%|███▏ | 99/311 [00:00<00:00, 254.36it/s, Materializing param=model.layers.8.self_attn.q_norm.weight] Loading weights: 32%|███▏ | 100/311 [00:00<00:00, 254.36it/s, Materializing param=model.layers.8.self_attn.q_proj.weight] Loading weights: 32%|███▏ | 100/311 [00:00<00:00, 254.36it/s, Materializing param=model.layers.8.self_attn.q_proj.weight] Loading weights: 5%|▌ | 16/311 [00:00<00:25, 11.53it/s, Materializing param=model.layers.1.mlp.gate_proj.weight] Loading weights: 5%|▌ | 16/311 [00:00<00:25, 11.53it/s, Materializing param=model.layers.1.mlp.gate_proj.weight] Loading weights: 15%|█▍ | 46/311 [00:00<00:02, 120.27it/s, Materializing param=model.layers.3.self_attn.v_proj.weight] Loading weights: 15%|█▍ | 46/311 [00:00<00:02, 120.27it/s, Materializing param=model.layers.3.self_attn.v_proj.weight] Loading weights: 31%|███ | 95/311 [00:00<00:01, 175.84it/s, Materializing param=model.layers.8.post_attention_layernorm.weight] Loading weights: 31%|███ | 95/311 [00:00<00:01, 175.84it/s, Materializing param=model.layers.8.post_attention_layernorm.weight] Loading weights: 31%|███ | 96/311 [00:00<00:01, 175.84it/s, Materializing param=model.layers.8.self_attn.k_norm.weight] Loading weights: 31%|███ | 96/311 [00:00<00:01, 175.84it/s, Materializing param=model.layers.8.self_attn.k_norm.weight] Loading weights: 31%|███ | 97/311 [00:00<00:01, 175.84it/s, Materializing param=model.layers.8.self_attn.k_proj.weight] Loading weights: 31%|███ | 97/311 [00:00<00:01, 175.84it/s, Materializing param=model.layers.8.self_attn.k_proj.weight] Loading weights: 32%|███▏ | 98/311 [00:00<00:01, 175.84it/s, Materializing param=model.layers.8.self_attn.o_proj.weight] Loading weights: 15%|█▌ | 47/311 [00:00<00:02, 120.27it/s, Materializing param=model.layers.4.input_layernorm.weight] Loading weights: 32%|███▏ | 98/311 [00:00<00:01, 175.84it/s, Materializing param=model.layers.8.self_attn.o_proj.weight] Loading weights: 15%|█▌ | 47/311 [00:00<00:02, 120.27it/s, Materializing param=model.layers.4.input_layernorm.weight] Loading weights: 32%|███▏ | 101/311 [00:00<00:00, 254.36it/s, Materializing param=model.layers.8.self_attn.v_proj.weight] Loading weights: 32%|███▏ | 101/311 [00:00<00:00, 254.36it/s, Materializing param=model.layers.8.self_attn.v_proj.weight] Loading weights: 15%|█▌ | 48/311 [00:00<00:02, 120.27it/s, Materializing param=model.layers.4.mlp.down_proj.weight] Loading weights: 15%|█▌ | 48/311 [00:00<00:02, 120.27it/s, Materializing param=model.layers.4.mlp.down_proj.weight] Loading weights: 33%|███▎ | 102/311 [00:00<00:00, 254.36it/s, Materializing param=model.layers.9.input_layernorm.weight] Loading weights: 33%|███▎ | 102/311 [00:00<00:00, 254.36it/s, Materializing param=model.layers.9.input_layernorm.weight] Loading weights: 32%|███▏ | 99/311 [00:00<00:01, 175.84it/s, Materializing param=model.layers.8.self_attn.q_norm.weight] Loading weights: 32%|███▏ | 99/311 [00:00<00:01, 175.84it/s, Materializing param=model.layers.8.self_attn.q_norm.weight] Loading weights: 32%|███▏ | 100/311 [00:00<00:01, 175.84it/s, Materializing param=model.layers.8.self_attn.q_proj.weight] Loading weights: 32%|███▏ | 100/311 [00:00<00:01, 175.84it/s, Materializing param=model.layers.8.self_attn.q_proj.weight] Loading weights: 28%|██▊ | 88/311 [00:00<00:00, 228.68it/s, Materializing param=model.layers.7.self_attn.q_norm.weight] Loading weights: 28%|██▊ | 88/311 [00:00<00:00, 228.68it/s, Materializing param=model.layers.7.self_attn.q_norm.weight] Loading weights: 29%|██▊ | 89/311 [00:00<00:00, 228.68it/s, Materializing param=model.layers.7.self_attn.q_proj.weight] Loading weights: 29%|██▊ | 89/311 [00:00<00:00, 228.68it/s, Materializing param=model.layers.7.self_attn.q_proj.weight] Loading weights: 29%|██▉ | 90/311 [00:00<00:00, 228.68it/s, Materializing param=model.layers.7.self_attn.v_proj.weight] Loading weights: 29%|██▉ | 90/311 [00:00<00:00, 228.68it/s, Materializing param=model.layers.7.self_attn.v_proj.weight] Loading weights: 32%|███▏ | 101/311 [00:00<00:01, 175.84it/s, Materializing param=model.layers.8.self_attn.v_proj.weight] Loading weights: 32%|███▏ | 101/311 [00:00<00:01, 175.84it/s, Materializing param=model.layers.8.self_attn.v_proj.weight] Loading weights: 29%|██▉ | 91/311 [00:00<00:00, 228.68it/s, Materializing param=model.layers.8.input_layernorm.weight] Loading weights: 29%|██▉ | 91/311 [00:00<00:00, 228.68it/s, Materializing param=model.layers.8.input_layernorm.weight] Loading weights: 30%|██▉ | 92/311 [00:00<00:00, 228.68it/s, Materializing param=model.layers.8.mlp.down_proj.weight] Loading weights: 30%|██▉ | 92/311 [00:00<00:00, 228.68it/s, Materializing param=model.layers.8.mlp.down_proj.weight] Loading weights: 33%|███▎ | 102/311 [00:00<00:01, 175.84it/s, Materializing param=model.layers.9.input_layernorm.weight] Loading weights: 33%|███▎ | 102/311 [00:00<00:01, 175.84it/s, Materializing param=model.layers.9.input_layernorm.weight] Loading weights: 33%|███▎ | 103/311 [00:00<00:01, 175.84it/s, Materializing param=model.layers.9.mlp.down_proj.weight] Loading weights: 33%|███▎ | 103/311 [00:00<00:01, 175.84it/s, Materializing param=model.layers.9.mlp.down_proj.weight] Loading weights: 20%|█▉ | 62/311 [00:00<00:02, 114.67it/s, Materializing param=model.layers.5.post_attention_layernorm.weight] Loading weights: 20%|█▉ | 62/311 [00:00<00:02, 114.67it/s, Materializing param=model.layers.5.post_attention_layernorm.weight] Loading weights: 26%|██▌ | 81/311 [00:00<00:01, 155.68it/s, Materializing param=model.layers.7.mlp.down_proj.weight] Loading weights: 26%|██▌ | 81/311 [00:00<00:01, 155.68it/s, Materializing param=model.layers.7.mlp.down_proj.weight] Loading weights: 5%|▌ | 17/311 [00:00<00:25, 11.53it/s, Materializing param=model.layers.1.mlp.up_proj.weight] Loading weights: 20%|██ | 63/311 [00:00<00:02, 114.67it/s, Materializing param=model.layers.5.self_attn.k_norm.weight] Loading weights: 5%|▌ | 17/311 [00:00<00:25, 11.53it/s, Materializing param=model.layers.1.mlp.up_proj.weight] Loading weights: 26%|██▋ | 82/311 [00:00<00:01, 155.68it/s, Materializing param=model.layers.7.mlp.gate_proj.weight] Loading weights: 20%|██ | 63/311 [00:00<00:02, 114.67it/s, Materializing param=model.layers.5.self_attn.k_norm.weight] Loading weights: 26%|██▋ | 82/311 [00:00<00:01, 155.68it/s, Materializing param=model.layers.7.mlp.gate_proj.weight] Loading weights: 21%|██ | 64/311 [00:00<00:02, 114.67it/s, Materializing param=model.layers.5.self_attn.k_proj.weight] Loading weights: 21%|██ | 64/311 [00:00<00:02, 114.67it/s, Materializing param=model.layers.5.self_attn.k_proj.weight] Loading weights: 27%|██▋ | 83/311 [00:00<00:01, 155.68it/s, Materializing param=model.layers.7.mlp.up_proj.weight] Loading weights: 27%|██▋ | 83/311 [00:00<00:01, 155.68it/s, Materializing param=model.layers.7.mlp.up_proj.weight] Loading weights: 21%|██ | 65/311 [00:00<00:02, 114.67it/s, Materializing param=model.layers.5.self_attn.o_proj.weight] Loading weights: 21%|██ | 65/311 [00:00<00:02, 114.67it/s, Materializing param=model.layers.5.self_attn.o_proj.weight] Loading weights: 16%|█▌ | 49/311 [00:00<00:02, 120.27it/s, Materializing param=model.layers.4.mlp.gate_proj.weight] Loading weights: 16%|█▌ | 49/311 [00:00<00:02, 120.27it/s, Materializing param=model.layers.4.mlp.gate_proj.weight] Loading weights: 30%|██▉ | 93/311 [00:00<00:00, 228.68it/s, Materializing param=model.layers.8.mlp.gate_proj.weight] Loading weights: 30%|██▉ | 93/311 [00:00<00:00, 228.68it/s, Materializing param=model.layers.8.mlp.gate_proj.weight] Loading weights: 27%|██▋ | 84/311 [00:00<00:01, 155.68it/s, Materializing param=model.layers.7.post_attention_layernorm.weight] Loading weights: 27%|██▋ | 84/311 [00:00<00:01, 155.68it/s, Materializing param=model.layers.7.post_attention_layernorm.weight] Loading weights: 27%|██▋ | 85/311 [00:00<00:01, 155.68it/s, Materializing param=model.layers.7.self_attn.k_norm.weight] Loading weights: 33%|███▎ | 103/311 [00:00<00:00, 254.36it/s, Materializing param=model.layers.9.mlp.down_proj.weight] Loading weights: 33%|███▎ | 104/311 [00:00<00:01, 175.84it/s, Materializing param=model.layers.9.mlp.gate_proj.weight] Loading weights: 27%|██▋ | 85/311 [00:00<00:01, 155.68it/s, Materializing param=model.layers.7.self_attn.k_norm.weight] Loading weights: 33%|███▎ | 104/311 [00:00<00:01, 175.84it/s, Materializing param=model.layers.9.mlp.gate_proj.weight] Loading weights: 28%|██▊ | 86/311 [00:00<00:01, 155.68it/s, Materializing param=model.layers.7.self_attn.k_proj.weight] Loading weights: 33%|███▎ | 103/311 [00:00<00:00, 254.36it/s, Materializing param=model.layers.9.mlp.down_proj.weight] Loading weights: 33%|███▎ | 104/311 [00:00<00:00, 254.36it/s, Materializing param=model.layers.9.mlp.gate_proj.weight] Loading weights: 33%|███▎ | 104/311 [00:00<00:00, 254.36it/s, Materializing param=model.layers.9.mlp.gate_proj.weight] Loading weights: 28%|██▊ | 86/311 [00:00<00:01, 155.68it/s, Materializing param=model.layers.7.self_attn.k_proj.weight] Loading weights: 34%|███▍ | 105/311 [00:00<00:00, 254.36it/s, Materializing param=model.layers.9.mlp.up_proj.weight] Loading weights: 21%|██ | 66/311 [00:00<00:02, 114.67it/s, Materializing param=model.layers.5.self_attn.q_norm.weight] Loading weights: 34%|███▍ | 105/311 [00:00<00:00, 254.36it/s, Materializing param=model.layers.9.mlp.up_proj.weight] Loading weights: 28%|██▊ | 87/311 [00:00<00:01, 155.68it/s, Materializing param=model.layers.7.self_attn.o_proj.weight] Loading weights: 21%|██ | 66/311 [00:00<00:02, 114.67it/s, Materializing param=model.layers.5.self_attn.q_norm.weight] Loading weights: 28%|██▊ | 87/311 [00:00<00:01, 155.68it/s, Materializing param=model.layers.7.self_attn.o_proj.weight] Loading weights: 22%|██▏ | 67/311 [00:00<00:02, 114.67it/s, Materializing param=model.layers.5.self_attn.q_proj.weight] Loading weights: 22%|██▏ | 67/311 [00:00<00:02, 114.67it/s, Materializing param=model.layers.5.self_attn.q_proj.weight] Loading weights: 22%|██▏ | 68/311 [00:00<00:02, 114.67it/s, Materializing param=model.layers.5.self_attn.v_proj.weight] Loading weights: 22%|██▏ | 68/311 [00:00<00:02, 114.67it/s, Materializing param=model.layers.5.self_attn.v_proj.weight] Loading weights: 6%|▌ | 18/311 [00:00<00:25, 11.53it/s, Materializing param=model.layers.1.post_attention_layernorm.weight] Loading weights: 6%|▌ | 18/311 [00:00<00:25, 11.53it/s, Materializing param=model.layers.1.post_attention_layernorm.weight] Loading weights: 6%|▌ | 19/311 [00:00<00:25, 11.53it/s, Materializing param=model.layers.1.self_attn.k_norm.weight] Loading weights: 6%|▌ | 19/311 [00:00<00:25, 11.53it/s, Materializing param=model.layers.1.self_attn.k_norm.weight] Loading weights: 6%|▋ | 20/311 [00:00<00:25, 11.53it/s, Materializing param=model.layers.1.self_attn.k_proj.weight] Loading weights: 6%|▋ | 20/311 [00:00<00:25, 11.53it/s, Materializing param=model.layers.1.self_attn.k_proj.weight] Loading weights: 22%|██▏ | 69/311 [00:00<00:02, 114.67it/s, Materializing param=model.layers.6.input_layernorm.weight] Loading weights: 22%|██▏ | 69/311 [00:00<00:02, 114.67it/s, Materializing param=model.layers.6.input_layernorm.weight] Loading weights: 7%|▋ | 21/311 [00:00<00:25, 11.53it/s, Materializing param=model.layers.1.self_attn.o_proj.weight] Loading weights: 7%|▋ | 21/311 [00:00<00:25, 11.53it/s, Materializing param=model.layers.1.self_attn.o_proj.weight] Loading weights: 28%|██▊ | 88/311 [00:00<00:01, 155.68it/s, Materializing param=model.layers.7.self_attn.q_norm.weight] Loading weights: 28%|██▊ | 88/311 [00:00<00:01, 155.68it/s, Materializing param=model.layers.7.self_attn.q_norm.weight] Loading weights: 29%|██▊ | 89/311 [00:00<00:01, 155.68it/s, Materializing param=model.layers.7.self_attn.q_proj.weight] Loading weights: 29%|██▊ | 89/311 [00:00<00:01, 155.68it/s, Materializing param=model.layers.7.self_attn.q_proj.weight] Loading weights: 16%|█▌ | 50/311 [00:00<00:02, 120.27it/s, Materializing param=model.layers.4.mlp.up_proj.weight] Loading weights: 16%|█▌ | 50/311 [00:00<00:02, 120.27it/s, Materializing param=model.layers.4.mlp.up_proj.weight] Loading weights: 30%|███ | 94/311 [00:00<00:00, 228.68it/s, Materializing param=model.layers.8.mlp.up_proj.weight] Loading weights: 30%|███ | 94/311 [00:00<00:00, 228.68it/s, Materializing param=model.layers.8.mlp.up_proj.weight] Loading weights: 7%|▋ | 22/311 [00:00<00:25, 11.53it/s, Materializing param=model.layers.1.self_attn.q_norm.weight] Loading weights: 7%|▋ | 22/311 [00:00<00:25, 11.53it/s, Materializing param=model.layers.1.self_attn.q_norm.weight] Loading weights: 29%|██▉ | 90/311 [00:00<00:01, 155.68it/s, Materializing param=model.layers.7.self_attn.v_proj.weight] Loading weights: 7%|▋ | 23/311 [00:00<00:24, 11.53it/s, Materializing param=model.layers.1.self_attn.q_proj.weight] Loading weights: 7%|▋ | 23/311 [00:00<00:24, 11.53it/s, Materializing param=model.layers.1.self_attn.q_proj.weight] Loading weights: 29%|██▉ | 90/311 [00:00<00:01, 155.68it/s, Materializing param=model.layers.7.self_attn.v_proj.weight] Loading weights: 29%|██▉ | 91/311 [00:00<00:01, 155.68it/s, Materializing param=model.layers.8.input_layernorm.weight] Loading weights: 29%|██▉ | 91/311 [00:00<00:01, 155.68it/s, Materializing param=model.layers.8.input_layernorm.weight] Loading weights: 30%|██▉ | 92/311 [00:00<00:01, 155.68it/s, Materializing param=model.layers.8.mlp.down_proj.weight] Loading weights: 34%|███▍ | 106/311 [00:00<00:00, 254.36it/s, Materializing param=model.layers.9.post_attention_layernorm.weight] Loading weights: 34%|███▍ | 106/311 [00:00<00:00, 254.36it/s, Materializing param=model.layers.9.post_attention_layernorm.weight] Loading weights: 30%|██▉ | 92/311 [00:00<00:01, 155.68it/s, Materializing param=model.layers.8.mlp.down_proj.weight] Loading weights: 34%|███▍ | 105/311 [00:00<00:00, 297.27it/s, Materializing param=model.layers.9.mlp.gate_proj.weight] Loading weights: 34%|███▍ | 107/311 [00:00<00:00, 254.36it/s, Materializing param=model.layers.9.self_attn.k_norm.weight] Loading weights: 34%|███▍ | 105/311 [00:00<00:00, 297.27it/s, Materializing param=model.layers.9.mlp.up_proj.weight] Loading weights: 34%|███▍ | 107/311 [00:00<00:00, 254.36it/s, Materializing param=model.layers.9.self_attn.k_norm.weight] Loading weights: 34%|███▍ | 105/311 [00:00<00:00, 297.27it/s, Materializing param=model.layers.9.mlp.up_proj.weight] Loading weights: 35%|███▍ | 108/311 [00:00<00:00, 254.36it/s, Materializing param=model.layers.9.self_attn.k_proj.weight] Loading weights: 35%|███▍ | 108/311 [00:00<00:00, 254.36it/s, Materializing param=model.layers.9.self_attn.k_proj.weight] Loading weights: 8%|▊ | 24/311 [00:00<00:24, 11.53it/s, Materializing param=model.layers.1.self_attn.v_proj.weight] Loading weights: 8%|▊ | 24/311 [00:00<00:24, 11.53it/s, Materializing param=model.layers.1.self_attn.v_proj.weight] Loading weights: 35%|███▌ | 109/311 [00:00<00:00, 254.36it/s, Materializing param=model.layers.9.self_attn.o_proj.weight] Loading weights: 35%|███▌ | 109/311 [00:00<00:00, 254.36it/s, Materializing param=model.layers.9.self_attn.o_proj.weight] Loading weights: 8%|▊ | 25/311 [00:00<00:24, 11.53it/s, Materializing param=model.layers.2.input_layernorm.weight] Loading weights: 8%|▊ | 25/311 [00:00<00:24, 11.53it/s, Materializing param=model.layers.2.input_layernorm.weight] Loading weights: 8%|▊ | 26/311 [00:00<00:24, 11.53it/s, Materializing param=model.layers.2.mlp.down_proj.weight] Loading weights: 8%|▊ | 26/311 [00:00<00:24, 11.53it/s, Materializing param=model.layers.2.mlp.down_proj.weight] Loading weights: 23%|██▎ | 70/311 [00:00<00:02, 114.67it/s, Materializing param=model.layers.6.mlp.down_proj.weight] Loading weights: 23%|██▎ | 70/311 [00:00<00:02, 114.67it/s, Materializing param=model.layers.6.mlp.down_proj.weight] Loading weights: 34%|███▍ | 106/311 [00:00<00:00, 297.27it/s, Materializing param=model.layers.9.post_attention_layernorm.weight] Loading weights: 31%|███ | 95/311 [00:00<00:00, 228.68it/s, Materializing param=model.layers.8.post_attention_layernorm.weight] Loading weights: 23%|██▎ | 71/311 [00:00<00:02, 114.67it/s, Materializing param=model.layers.6.mlp.gate_proj.weight] Loading weights: 34%|███▍ | 106/311 [00:00<00:00, 297.27it/s, Materializing param=model.layers.9.post_attention_layernorm.weight] Loading weights: 23%|██▎ | 71/311 [00:00<00:02, 114.67it/s, Materializing param=model.layers.6.mlp.gate_proj.weight] Loading weights: 31%|███ | 95/311 [00:00<00:00, 228.68it/s, Materializing param=model.layers.8.post_attention_layernorm.weight] Loading weights: 16%|█▋ | 51/311 [00:00<00:02, 120.27it/s, Materializing param=model.layers.4.post_attention_layernorm.weight] Loading weights: 31%|███ | 96/311 [00:00<00:00, 228.68it/s, Materializing param=model.layers.8.self_attn.k_norm.weight] Loading weights: 34%|███▍ | 107/311 [00:00<00:00, 297.27it/s, Materializing param=model.layers.9.self_attn.k_norm.weight] Loading weights: 30%|██▉ | 93/311 [00:00<00:00, 259.35it/s, Materializing param=model.layers.8.mlp.down_proj.weight] Loading weights: 31%|███ | 96/311 [00:00<00:00, 228.68it/s, Materializing param=model.layers.8.self_attn.k_norm.weight] Loading weights: 16%|█▋ | 51/311 [00:00<00:02, 120.27it/s, Materializing param=model.layers.4.post_attention_layernorm.weight] Loading weights: 34%|███▍ | 107/311 [00:00<00:00, 297.27it/s, Materializing param=model.layers.9.self_attn.k_norm.weight] Loading weights: 17%|█▋ | 52/311 [00:00<00:02, 120.27it/s, Materializing param=model.layers.4.self_attn.k_norm.weight] Loading weights: 30%|██▉ | 93/311 [00:00<00:00, 259.35it/s, Materializing param=model.layers.8.mlp.gate_proj.weight] Loading weights: 31%|███ | 97/311 [00:00<00:00, 228.68it/s, Materializing param=model.layers.8.self_attn.k_proj.weight] Loading weights: 35%|███▍ | 108/311 [00:00<00:00, 297.27it/s, Materializing param=model.layers.9.self_attn.k_proj.weight] Loading weights: 17%|█▋ | 52/311 [00:00<00:02, 120.27it/s, Materializing param=model.layers.4.self_attn.k_norm.weight] Loading weights: 35%|███▍ | 108/311 [00:00<00:00, 297.27it/s, Materializing param=model.layers.9.self_attn.k_proj.weight] Loading weights: 30%|██▉ | 93/311 [00:00<00:00, 259.35it/s, Materializing param=model.layers.8.mlp.gate_proj.weight] Loading weights: 31%|███ | 97/311 [00:00<00:00, 228.68it/s, Materializing param=model.layers.8.self_attn.k_proj.weight] Loading weights: 17%|█▋ | 53/311 [00:00<00:02, 120.27it/s, Materializing param=model.layers.4.self_attn.k_proj.weight] Loading weights: 35%|███▌ | 109/311 [00:00<00:00, 297.27it/s, Materializing param=model.layers.9.self_attn.o_proj.weight] Loading weights: 17%|█▋ | 53/311 [00:00<00:02, 120.27it/s, Materializing param=model.layers.4.self_attn.k_proj.weight] Loading weights: 35%|███▌ | 109/311 [00:00<00:00, 297.27it/s, Materializing param=model.layers.9.self_attn.o_proj.weight] Loading weights: 32%|███▏ | 98/311 [00:00<00:00, 228.68it/s, Materializing param=model.layers.8.self_attn.o_proj.weight] Loading weights: 32%|███▏ | 98/311 [00:00<00:00, 228.68it/s, Materializing param=model.layers.8.self_attn.o_proj.weight] Loading weights: 35%|███▌ | 110/311 [00:00<00:00, 254.36it/s, Materializing param=model.layers.9.self_attn.q_norm.weight] Loading weights: 35%|███▌ | 110/311 [00:00<00:00, 254.36it/s, Materializing param=model.layers.9.self_attn.q_norm.weight] Loading weights: 17%|█▋ | 54/311 [00:00<00:02, 120.27it/s, Materializing param=model.layers.4.self_attn.o_proj.weight] Loading weights: 17%|█▋ | 54/311 [00:00<00:02, 120.27it/s, Materializing param=model.layers.4.self_attn.o_proj.weight] Loading weights: 36%|███▌ | 111/311 [00:00<00:00, 254.36it/s, Materializing param=model.layers.9.self_attn.q_proj.weight] Loading weights: 36%|███▌ | 111/311 [00:00<00:00, 254.36it/s, Materializing param=model.layers.9.self_attn.q_proj.weight] Loading weights: 36%|███▌ | 112/311 [00:00<00:00, 254.36it/s, Materializing param=model.layers.9.self_attn.v_proj.weight] Loading weights: 36%|███▌ | 112/311 [00:00<00:00, 254.36it/s, Materializing param=model.layers.9.self_attn.v_proj.weight] Loading weights: 36%|███▋ | 113/311 [00:00<00:00, 254.36it/s, Materializing param=model.layers.10.input_layernorm.weight] Loading weights: 36%|███▋ | 113/311 [00:00<00:00, 254.36it/s, Materializing param=model.layers.10.input_layernorm.weight] Loading weights: 37%|███▋ | 114/311 [00:00<00:00, 254.36it/s, Materializing param=model.layers.10.mlp.down_proj.weight] Loading weights: 35%|███▌ | 110/311 [00:00<00:00, 297.27it/s, Materializing param=model.layers.9.self_attn.q_norm.weight] Loading weights: 37%|███▋ | 114/311 [00:00<00:00, 254.36it/s, Materializing param=model.layers.10.mlp.down_proj.weight] Loading weights: 35%|███▌ | 110/311 [00:00<00:00, 297.27it/s, Materializing param=model.layers.9.self_attn.q_norm.weight] Loading weights: 36%|███▌ | 111/311 [00:00<00:00, 297.27it/s, Materializing param=model.layers.9.self_attn.q_proj.weight] Loading weights: 36%|███▌ | 111/311 [00:00<00:00, 297.27it/s, Materializing param=model.layers.9.self_attn.q_proj.weight] Loading weights: 32%|███▏ | 99/311 [00:00<00:00, 228.68it/s, Materializing param=model.layers.8.self_attn.q_norm.weight] Loading weights: 32%|███▏ | 99/311 [00:00<00:00, 228.68it/s, Materializing param=model.layers.8.self_attn.q_norm.weight] Loading weights: 32%|███▏ | 100/311 [00:00<00:00, 228.68it/s, Materializing param=model.layers.8.self_attn.q_proj.weight] Loading weights: 32%|███▏ | 100/311 [00:00<00:00, 228.68it/s, Materializing param=model.layers.8.self_attn.q_proj.weight] Loading weights: 36%|███▌ | 112/311 [00:00<00:00, 297.27it/s, Materializing param=model.layers.9.self_attn.v_proj.weight] Loading weights: 36%|███▌ | 112/311 [00:00<00:00, 297.27it/s, Materializing param=model.layers.9.self_attn.v_proj.weight] Loading weights: 18%|█▊ | 55/311 [00:00<00:02, 120.27it/s, Materializing param=model.layers.4.self_attn.q_norm.weight] Loading weights: 18%|█▊ | 55/311 [00:00<00:02, 120.27it/s, Materializing param=model.layers.4.self_attn.q_norm.weight] Loading weights: 18%|█▊ | 56/311 [00:00<00:02, 120.27it/s, Materializing param=model.layers.4.self_attn.q_proj.weight] Loading weights: 18%|█▊ | 56/311 [00:00<00:02, 120.27it/s, Materializing param=model.layers.4.self_attn.q_proj.weight] Loading weights: 36%|███▋ | 113/311 [00:00<00:00, 297.27it/s, Materializing param=model.layers.10.input_layernorm.weight] Loading weights: 36%|███▋ | 113/311 [00:00<00:00, 297.27it/s, Materializing param=model.layers.10.input_layernorm.weight] Loading weights: 9%|▊ | 27/311 [00:00<00:24, 11.53it/s, Materializing param=model.layers.2.mlp.gate_proj.weight] Loading weights: 9%|▊ | 27/311 [00:00<00:24, 11.53it/s, Materializing param=model.layers.2.mlp.gate_proj.weight] Loading weights: 32%|███▏ | 101/311 [00:00<00:00, 228.68it/s, Materializing param=model.layers.8.self_attn.v_proj.weight] Loading weights: 32%|███▏ | 101/311 [00:00<00:00, 228.68it/s, Materializing param=model.layers.8.self_attn.v_proj.weight] Loading weights: 37%|███▋ | 115/311 [00:00<00:00, 254.36it/s, Materializing param=model.layers.10.mlp.gate_proj.weight] Loading weights: 37%|███▋ | 115/311 [00:00<00:00, 254.36it/s, Materializing param=model.layers.10.mlp.gate_proj.weight] Loading weights: 18%|█▊ | 57/311 [00:00<00:02, 120.27it/s, Materializing param=model.layers.4.self_attn.v_proj.weight] Loading weights: 18%|█▊ | 57/311 [00:00<00:02, 120.27it/s, Materializing param=model.layers.4.self_attn.v_proj.weight] Loading weights: 33%|███▎ | 102/311 [00:00<00:00, 228.68it/s, Materializing param=model.layers.9.input_layernorm.weight] Loading weights: 33%|███▎ | 102/311 [00:00<00:00, 228.68it/s, Materializing param=model.layers.9.input_layernorm.weight] Loading weights: 23%|██▎ | 72/311 [00:00<00:01, 182.18it/s, Materializing param=model.layers.6.mlp.gate_proj.weight] Loading weights: 33%|███▎ | 103/311 [00:00<00:00, 228.68it/s, Materializing param=model.layers.9.mlp.down_proj.weight] Loading weights: 23%|██▎ | 72/311 [00:00<00:01, 182.18it/s, Materializing param=model.layers.6.mlp.up_proj.weight] Loading weights: 33%|███▎ | 103/311 [00:00<00:00, 228.68it/s, Materializing param=model.layers.9.mlp.down_proj.weight] Loading weights: 23%|██▎ | 72/311 [00:00<00:01, 182.18it/s, Materializing param=model.layers.6.mlp.up_proj.weight] Loading weights: 30%|███ | 94/311 [00:00<00:00, 259.35it/s, Materializing param=model.layers.8.mlp.up_proj.weight] Loading weights: 30%|███ | 94/311 [00:00<00:00, 259.35it/s, Materializing param=model.layers.8.mlp.up_proj.weight] Loading weights: 19%|█▊ | 58/311 [00:00<00:02, 120.27it/s, Materializing param=model.layers.5.input_layernorm.weight] Loading weights: 19%|█▊ | 58/311 [00:00<00:02, 120.27it/s, Materializing param=model.layers.5.input_layernorm.weight] Loading weights: 19%|█▉ | 59/311 [00:00<00:02, 120.27it/s, Materializing param=model.layers.5.mlp.down_proj.weight] Loading weights: 19%|█▉ | 59/311 [00:00<00:02, 120.27it/s, Materializing param=model.layers.5.mlp.down_proj.weight] Loading weights: 37%|███▋ | 114/311 [00:00<00:00, 297.27it/s, Materializing param=model.layers.10.mlp.down_proj.weight] Loading weights: 37%|███▋ | 114/311 [00:00<00:00, 297.27it/s, Materializing param=model.layers.10.mlp.down_proj.weight] Loading weights: 37%|███▋ | 115/311 [00:00<00:00, 297.27it/s, Materializing param=model.layers.10.mlp.gate_proj.weight] Loading weights: 37%|███▋ | 115/311 [00:00<00:00, 297.27it/s, Materializing param=model.layers.10.mlp.gate_proj.weight] Loading weights: 9%|▉ | 28/311 [00:00<00:24, 11.53it/s, Materializing param=model.layers.2.mlp.up_proj.weight] Loading weights: 9%|▉ | 28/311 [00:00<00:24, 11.53it/s, Materializing param=model.layers.2.mlp.up_proj.weight] Loading weights: 37%|███▋ | 116/311 [00:00<00:00, 254.36it/s, Materializing param=model.layers.10.mlp.up_proj.weight] Loading weights: 37%|███▋ | 116/311 [00:00<00:00, 254.36it/s, Materializing param=model.layers.10.mlp.up_proj.weight] Loading weights: 33%|███▎ | 104/311 [00:00<00:00, 228.68it/s, Materializing param=model.layers.9.mlp.gate_proj.weight] Loading weights: 33%|███▎ | 104/311 [00:00<00:00, 228.68it/s, Materializing param=model.layers.9.mlp.gate_proj.weight][INFO|2026-04-29 17:06:08] llamafactory.model.loader:144 >> trainable params: 2,031,739,904 || all params: 2,031,739,904 || trainable%: 100.0000 + Loading weights: 31%|███ | 95/311 [00:00<00:00, 259.35it/s, Materializing param=model.layers.8.post_attention_layernorm.weight] Loading weights: 31%|███ | 95/311 [00:00<00:00, 259.35it/s, Materializing param=model.layers.8.post_attention_layernorm.weight] Loading weights: 31%|███ | 96/311 [00:00<00:00, 259.35it/s, Materializing param=model.layers.8.self_attn.k_norm.weight] Loading weights: 31%|███ | 96/311 [00:00<00:00, 259.35it/s, Materializing param=model.layers.8.self_attn.k_norm.weight] Loading weights: 37%|███▋ | 116/311 [00:00<00:00, 297.27it/s, Materializing param=model.layers.10.mlp.up_proj.weight] Loading weights: 37%|███▋ | 116/311 [00:00<00:00, 297.27it/s, Materializing param=model.layers.10.mlp.up_proj.weight] Loading weights: 31%|███ | 97/311 [00:00<00:00, 259.35it/s, Materializing param=model.layers.8.self_attn.k_proj.weight] Loading weights: 31%|███ | 97/311 [00:00<00:00, 259.35it/s, Materializing param=model.layers.8.self_attn.k_proj.weight] Loading weights: 23%|██▎ | 73/311 [00:00<00:01, 182.18it/s, Materializing param=model.layers.6.post_attention_layernorm.weight] Loading weights: 23%|██▎ | 73/311 [00:00<00:01, 182.18it/s, Materializing param=model.layers.6.post_attention_layernorm.weight] Loading weights: 32%|███▏ | 98/311 [00:00<00:00, 259.35it/s, Materializing param=model.layers.8.self_attn.o_proj.weight] Loading weights: 32%|███▏ | 98/311 [00:00<00:00, 259.35it/s, Materializing param=model.layers.8.self_attn.o_proj.weight] Loading weights: 24%|██▍ | 74/311 [00:00<00:01, 182.18it/s, Materializing param=model.layers.6.self_attn.k_norm.weight] Loading weights: 24%|██▍ | 74/311 [00:00<00:01, 182.18it/s, Materializing param=model.layers.6.self_attn.k_norm.weight] Loading weights: 19%|█▉ | 60/311 [00:00<00:02, 120.27it/s, Materializing param=model.layers.5.mlp.gate_proj.weight] Loading weights: 24%|██▍ | 75/311 [00:00<00:01, 182.18it/s, Materializing param=model.layers.6.self_attn.k_proj.weight] Loading weights: 19%|█▉ | 60/311 [00:00<00:02, 120.27it/s, Materializing param=model.layers.5.mlp.gate_proj.weight] Loading weights: 24%|██▍ | 75/311 [00:00<00:01, 182.18it/s, Materializing param=model.layers.6.self_attn.k_proj.weight] Loading weights: 24%|██▍ | 76/311 [00:00<00:01, 182.18it/s, Materializing param=model.layers.6.self_attn.o_proj.weight] Loading weights: 24%|██▍ | 76/311 [00:00<00:01, 182.18it/s, Materializing param=model.layers.6.self_attn.o_proj.weight] Loading weights: 32%|███▏ | 99/311 [00:00<00:00, 259.35it/s, Materializing param=model.layers.8.self_attn.q_norm.weight] Loading weights: 32%|███▏ | 99/311 [00:00<00:00, 259.35it/s, Materializing param=model.layers.8.self_attn.q_norm.weight] Loading weights: 32%|███▏ | 100/311 [00:00<00:00, 259.35it/s, Materializing param=model.layers.8.self_attn.q_proj.weight] Loading weights: 32%|███▏ | 100/311 [00:00<00:00, 259.35it/s, Materializing param=model.layers.8.self_attn.q_proj.weight] Loading weights: 25%|██▍ | 77/311 [00:00<00:01, 182.18it/s, Materializing param=model.layers.6.self_attn.q_norm.weight] Loading weights: 25%|██▍ | 77/311 [00:00<00:01, 182.18it/s, Materializing param=model.layers.6.self_attn.q_norm.weight] Loading weights: 25%|██▌ | 78/311 [00:00<00:01, 182.18it/s, Materializing param=model.layers.6.self_attn.q_proj.weight] Loading weights: 25%|██▌ | 78/311 [00:00<00:01, 182.18it/s, Materializing param=model.layers.6.self_attn.q_proj.weight] Loading weights: 32%|███▏ | 101/311 [00:00<00:00, 259.35it/s, Materializing param=model.layers.8.self_attn.v_proj.weight] Loading weights: 32%|███▏ | 101/311 [00:00<00:00, 259.35it/s, Materializing param=model.layers.8.self_attn.v_proj.weight] Loading weights: 38%|███▊ | 117/311 [00:00<00:00, 254.36it/s, Materializing param=model.layers.10.post_attention_layernorm.weight] Loading weights: 38%|███▊ | 117/311 [00:00<00:00, 297.27it/s, Materializing param=model.layers.10.post_attention_layernorm.weight] Loading weights: 34%|███▍ | 105/311 [00:00<00:00, 228.68it/s, Materializing param=model.layers.9.mlp.up_proj.weight] Loading weights: 38%|███▊ | 117/311 [00:00<00:00, 254.36it/s, Materializing param=model.layers.10.post_attention_layernorm.weight] Loading weights: 9%|▉ | 29/311 [00:00<00:24, 11.53it/s, Materializing param=model.layers.2.post_attention_layernorm.weight] Loading weights: 34%|███▍ | 105/311 [00:00<00:00, 228.68it/s, Materializing param=model.layers.9.mlp.up_proj.weight] Loading weights: 38%|███▊ | 118/311 [00:00<00:00, 254.36it/s, Materializing param=model.layers.10.self_attn.k_norm.weight] Loading weights: 38%|███▊ | 117/311 [00:00<00:00, 297.27it/s, Materializing param=model.layers.10.post_attention_layernorm.weight] Loading weights: 38%|███▊ | 118/311 [00:00<00:00, 254.36it/s, Materializing param=model.layers.10.self_attn.k_norm.weight] Loading weights: 9%|▉ | 29/311 [00:00<00:24, 11.53it/s, Materializing param=model.layers.2.post_attention_layernorm.weight] Loading weights: 38%|███▊ | 119/311 [00:00<00:00, 254.36it/s, Materializing param=model.layers.10.self_attn.k_proj.weight] Loading weights: 38%|███▊ | 118/311 [00:00<00:00, 297.27it/s, Materializing param=model.layers.10.self_attn.k_norm.weight] Loading weights: 38%|███▊ | 119/311 [00:00<00:00, 254.36it/s, Materializing param=model.layers.10.self_attn.k_proj.weight] Loading weights: 38%|███▊ | 118/311 [00:00<00:00, 297.27it/s, Materializing param=model.layers.10.self_attn.k_norm.weight] Loading weights: 10%|▉ | 30/311 [00:00<00:24, 11.53it/s, Materializing param=model.layers.2.self_attn.k_norm.weight] Loading weights: 10%|▉ | 30/311 [00:00<00:24, 11.53it/s, Materializing param=model.layers.2.self_attn.k_norm.weight] Loading weights: 10%|▉ | 31/311 [00:00<00:24, 11.53it/s, Materializing param=model.layers.2.self_attn.k_proj.weight] Loading weights: 38%|███▊ | 119/311 [00:00<00:00, 297.27it/s, Materializing param=model.layers.10.self_attn.k_proj.weight] Loading weights: 10%|▉ | 31/311 [00:00<00:24, 11.53it/s, Materializing param=model.layers.2.self_attn.k_proj.weight] Loading weights: 38%|███▊ | 119/311 [00:00<00:00, 297.27it/s, Materializing param=model.layers.10.self_attn.k_proj.weight] Loading weights: 33%|███▎ | 102/311 [00:00<00:00, 259.35it/s, Materializing param=model.layers.9.input_layernorm.weight] Loading weights: 39%|███▊ | 120/311 [00:00<00:00, 254.36it/s, Materializing param=model.layers.10.self_attn.o_proj.weight] Loading weights: 39%|███▊ | 120/311 [00:00<00:00, 254.36it/s, Materializing param=model.layers.10.self_attn.o_proj.weight] Loading weights: 33%|███▎ | 102/311 [00:00<00:00, 259.35it/s, Materializing param=model.layers.9.input_layernorm.weight] Loading weights: 39%|███▊ | 120/311 [00:00<00:00, 297.27it/s, Materializing param=model.layers.10.self_attn.o_proj.weight] Loading weights: 39%|███▊ | 120/311 [00:00<00:00, 297.27it/s, Materializing param=model.layers.10.self_attn.o_proj.weight] Loading weights: 33%|███▎ | 103/311 [00:00<00:00, 259.35it/s, Materializing param=model.layers.9.mlp.down_proj.weight] Loading weights: 33%|███▎ | 103/311 [00:00<00:00, 259.35it/s, Materializing param=model.layers.9.mlp.down_proj.weight] Loading weights: 10%|█ | 32/311 [00:00<00:24, 11.53it/s, Materializing param=model.layers.2.self_attn.o_proj.weight] Loading weights: 10%|█ | 32/311 [00:00<00:24, 11.53it/s, Materializing param=model.layers.2.self_attn.o_proj.weight] Loading weights: 25%|██▌ | 79/311 [00:00<00:01, 182.18it/s, Materializing param=model.layers.6.self_attn.v_proj.weight] Loading weights: 25%|██▌ | 79/311 [00:00<00:01, 182.18it/s, Materializing param=model.layers.6.self_attn.v_proj.weight] Loading weights: 39%|███▉ | 121/311 [00:00<00:00, 254.36it/s, Materializing param=model.layers.10.self_attn.q_norm.weight] Loading weights: 39%|███▉ | 121/311 [00:00<00:00, 297.27it/s, Materializing param=model.layers.10.self_attn.q_norm.weight] Loading weights: 39%|███▉ | 121/311 [00:00<00:00, 254.36it/s, Materializing param=model.layers.10.self_attn.q_norm.weight] Loading weights: 39%|███▉ | 121/311 [00:00<00:00, 297.27it/s, Materializing param=model.layers.10.self_attn.q_norm.weight] Loading weights: 26%|██▌ | 80/311 [00:00<00:01, 182.18it/s, Materializing param=model.layers.7.input_layernorm.weight] Loading weights: 39%|███▉ | 122/311 [00:00<00:00, 297.27it/s, Materializing param=model.layers.10.self_attn.q_proj.weight] Loading weights: 20%|█▉ | 61/311 [00:00<00:02, 120.27it/s, Materializing param=model.layers.5.mlp.up_proj.weight] Loading weights: 26%|██▌ | 80/311 [00:00<00:01, 182.18it/s, Materializing param=model.layers.7.input_layernorm.weight] Loading weights: 39%|███▉ | 122/311 [00:00<00:00, 297.27it/s, Materializing param=model.layers.10.self_attn.q_proj.weight] Loading weights: 20%|█▉ | 61/311 [00:00<00:02, 120.27it/s, Materializing param=model.layers.5.mlp.up_proj.weight] Loading weights: 26%|██▌ | 81/311 [00:00<00:01, 182.18it/s, Materializing param=model.layers.7.mlp.down_proj.weight] Loading weights: 26%|██▌ | 81/311 [00:00<00:01, 182.18it/s, Materializing param=model.layers.7.mlp.down_proj.weight] Loading weights: 11%|█ | 33/311 [00:00<00:24, 11.53it/s, Materializing param=model.layers.2.self_attn.q_norm.weight] Loading weights: 11%|█ | 33/311 [00:00<00:24, 11.53it/s, Materializing param=model.layers.2.self_attn.q_norm.weight] Loading weights: 11%|█ | 34/311 [00:00<00:24, 11.53it/s, Materializing param=model.layers.2.self_attn.q_proj.weight] Loading weights: 11%|█ | 34/311 [00:00<00:24, 11.53it/s, Materializing param=model.layers.2.self_attn.q_proj.weight] Loading weights: 40%|███▉ | 123/311 [00:00<00:00, 297.27it/s, Materializing param=model.layers.10.self_attn.v_proj.weight] Loading weights: 40%|███▉ | 123/311 [00:00<00:00, 297.27it/s, Materializing param=model.layers.10.self_attn.v_proj.weight] Loading weights: 39%|███▉ | 122/311 [00:00<00:00, 254.36it/s, Materializing param=model.layers.10.self_attn.q_proj.weight] Loading weights: 39%|███▉ | 122/311 [00:00<00:00, 254.36it/s, Materializing param=model.layers.10.self_attn.q_proj.weight] Loading weights: 40%|███▉ | 123/311 [00:00<00:00, 254.36it/s, Materializing param=model.layers.10.self_attn.v_proj.weight] Loading weights: 40%|███▉ | 123/311 [00:00<00:00, 254.36it/s, Materializing param=model.layers.10.self_attn.v_proj.weight] Loading weights: 11%|█▏ | 35/311 [00:00<00:23, 11.53it/s, Materializing param=model.layers.2.self_attn.v_proj.weight] Loading weights: 33%|███▎ | 104/311 [00:00<00:00, 259.35it/s, Materializing param=model.layers.9.mlp.gate_proj.weight] Loading weights: 11%|█▏ | 35/311 [00:00<00:23, 11.53it/s, Materializing param=model.layers.2.self_attn.v_proj.weight] Loading weights: 40%|███▉ | 124/311 [00:00<00:00, 254.36it/s, Materializing param=model.layers.11.input_layernorm.weight] Loading weights: 33%|███▎ | 104/311 [00:00<00:00, 259.35it/s, Materializing param=model.layers.9.mlp.gate_proj.weight] Loading weights: 40%|███▉ | 124/311 [00:00<00:00, 254.36it/s, Materializing param=model.layers.11.input_layernorm.weight] Loading weights: 40%|███▉ | 124/311 [00:00<00:00, 297.27it/s, Materializing param=model.layers.11.input_layernorm.weight] Loading weights: 40%|███▉ | 124/311 [00:00<00:00, 297.27it/s, Materializing param=model.layers.11.input_layernorm.weight] Loading weights: 40%|████ | 125/311 [00:00<00:00, 254.36it/s, Materializing param=model.layers.11.mlp.down_proj.weight] Loading weights: 40%|████ | 125/311 [00:00<00:00, 297.27it/s, Materializing param=model.layers.11.mlp.down_proj.weight] Loading weights: 40%|████ | 125/311 [00:00<00:00, 254.36it/s, Materializing param=model.layers.11.mlp.down_proj.weight] Loading weights: 34%|███▍ | 106/311 [00:00<00:00, 228.68it/s, Materializing param=model.layers.9.post_attention_layernorm.weight] Loading weights: 40%|████ | 125/311 [00:00<00:00, 297.27it/s, Materializing param=model.layers.11.mlp.down_proj.weight] Loading weights: 34%|███▍ | 106/311 [00:00<00:00, 228.68it/s, Materializing param=model.layers.9.post_attention_layernorm.weight] Loading weights: 34%|███▍ | 107/311 [00:00<00:00, 228.68it/s, Materializing param=model.layers.9.self_attn.k_norm.weight] Loading weights: 34%|███▍ | 107/311 [00:00<00:00, 228.68it/s, Materializing param=model.layers.9.self_attn.k_norm.weight] Loading weights: 35%|███▍ | 108/311 [00:00<00:00, 228.68it/s, Materializing param=model.layers.9.self_attn.k_proj.weight] Loading weights: 35%|███▍ | 108/311 [00:00<00:00, 228.68it/s, Materializing param=model.layers.9.self_attn.k_proj.weight] Loading weights: 12%|█▏ | 36/311 [00:00<00:23, 11.53it/s, Materializing param=model.layers.3.input_layernorm.weight] Loading weights: 12%|█▏ | 36/311 [00:00<00:23, 11.53it/s, Materializing param=model.layers.3.input_layernorm.weight] Loading weights: 12%|█▏ | 37/311 [00:00<00:23, 11.53it/s, Materializing param=model.layers.3.mlp.down_proj.weight] Loading weights: 12%|█▏ | 37/311 [00:00<00:23, 11.53it/s, Materializing param=model.layers.3.mlp.down_proj.weight] Loading weights: 35%|███▌ | 109/311 [00:00<00:00, 228.68it/s, Materializing param=model.layers.9.self_attn.o_proj.weight] Loading weights: 35%|███▌ | 109/311 [00:00<00:00, 228.68it/s, Materializing param=model.layers.9.self_attn.o_proj.weight] Loading weights: 26%|██▋ | 82/311 [00:00<00:01, 182.18it/s, Materializing param=model.layers.7.mlp.gate_proj.weight] Loading weights: 26%|██▋ | 82/311 [00:00<00:01, 182.18it/s, Materializing param=model.layers.7.mlp.gate_proj.weight] Loading weights: 35%|███▌ | 110/311 [00:00<00:00, 228.68it/s, Materializing param=model.layers.9.self_attn.q_norm.weight] Loading weights: 35%|███▌ | 110/311 [00:00<00:00, 228.68it/s, Materializing param=model.layers.9.self_attn.q_norm.weight] Loading weights: 36%|███▌ | 111/311 [00:00<00:00, 228.68it/s, Materializing param=model.layers.9.self_attn.q_proj.weight] Loading weights: 36%|███▌ | 111/311 [00:00<00:00, 228.68it/s, Materializing param=model.layers.9.self_attn.q_proj.weight] Loading weights: 20%|█▉ | 62/311 [00:00<00:02, 120.27it/s, Materializing param=model.layers.5.post_attention_layernorm.weight] Loading weights: 20%|█▉ | 62/311 [00:00<00:02, 120.27it/s, Materializing param=model.layers.5.post_attention_layernorm.weight] Loading weights: 20%|██ | 63/311 [00:00<00:02, 120.27it/s, Materializing param=model.layers.5.self_attn.k_norm.weight] Loading weights: 20%|██ | 63/311 [00:00<00:02, 120.27it/s, Materializing param=model.layers.5.self_attn.k_norm.weight] Loading weights: 21%|██ | 64/311 [00:00<00:02, 120.27it/s, Materializing param=model.layers.5.self_attn.k_proj.weight] Loading weights: 21%|██ | 64/311 [00:00<00:02, 120.27it/s, Materializing param=model.layers.5.self_attn.k_proj.weight] Loading weights: 34%|███▍ | 105/311 [00:00<00:00, 259.35it/s, Materializing param=model.layers.9.mlp.up_proj.weight] Loading weights: 21%|██ | 65/311 [00:00<00:02, 120.27it/s, Materializing param=model.layers.5.self_attn.o_proj.weight] Loading weights: 21%|██ | 65/311 [00:00<00:02, 120.27it/s, Materializing param=model.layers.5.self_attn.o_proj.weight] Loading weights: 34%|███▍ | 105/311 [00:00<00:00, 259.35it/s, Materializing param=model.layers.9.mlp.up_proj.weight] Loading weights: 36%|███▌ | 112/311 [00:00<00:00, 228.68it/s, Materializing param=model.layers.9.self_attn.v_proj.weight] Loading weights: 36%|███▌ | 112/311 [00:00<00:00, 228.68it/s, Materializing param=model.layers.9.self_attn.v_proj.weight] Loading weights: 41%|████ | 126/311 [00:00<00:00, 254.36it/s, Materializing param=model.layers.11.mlp.gate_proj.weight] Loading weights: 41%|████ | 126/311 [00:00<00:00, 254.36it/s, Materializing param=model.layers.11.mlp.gate_proj.weight] Loading weights: 41%|████ | 126/311 [00:00<00:00, 297.27it/s, Materializing param=model.layers.11.mlp.gate_proj.weight] Loading weights: 41%|████ | 126/311 [00:00<00:00, 297.27it/s, Materializing param=model.layers.11.mlp.gate_proj.weight] Loading weights: 36%|███▋ | 113/311 [00:00<00:00, 228.68it/s, Materializing param=model.layers.10.input_layernorm.weight] Loading weights: 36%|███▋ | 113/311 [00:00<00:00, 228.68it/s, Materializing param=model.layers.10.input_layernorm.weight] Loading weights: 37%|███▋ | 114/311 [00:00<00:00, 228.68it/s, Materializing param=model.layers.10.mlp.down_proj.weight] Loading weights: 37%|███▋ | 114/311 [00:00<00:00, 228.68it/s, Materializing param=model.layers.10.mlp.down_proj.weight] Loading weights: 21%|██ | 66/311 [00:00<00:02, 120.27it/s, Materializing param=model.layers.5.self_attn.q_norm.weight] Loading weights: 21%|██ | 66/311 [00:00<00:02, 120.27it/s, Materializing param=model.layers.5.self_attn.q_norm.weight] Loading weights: 12%|█▏ | 38/311 [00:00<00:23, 11.53it/s, Materializing param=model.layers.3.mlp.gate_proj.weight] Loading weights: 12%|█▏ | 38/311 [00:00<00:23, 11.53it/s, Materializing param=model.layers.3.mlp.gate_proj.weight] Loading weights: 22%|██▏ | 67/311 [00:00<00:02, 120.27it/s, Materializing param=model.layers.5.self_attn.q_proj.weight] Loading weights: 22%|██▏ | 67/311 [00:00<00:02, 120.27it/s, Materializing param=model.layers.5.self_attn.q_proj.weight] Loading weights: 22%|██▏ | 68/311 [00:00<00:02, 120.27it/s, Materializing param=model.layers.5.self_attn.v_proj.weight] Loading weights: 22%|██▏ | 68/311 [00:00<00:02, 120.27it/s, Materializing param=model.layers.5.self_attn.v_proj.weight] Loading weights: 34%|███▍ | 106/311 [00:00<00:00, 259.35it/s, Materializing param=model.layers.9.post_attention_layernorm.weight] Loading weights: 27%|██▋ | 83/311 [00:00<00:01, 182.18it/s, Materializing param=model.layers.7.mlp.up_proj.weight] Loading weights: 22%|██▏ | 69/311 [00:00<00:02, 120.27it/s, Materializing param=model.layers.6.input_layernorm.weight] Loading weights: 27%|██▋ | 83/311 [00:00<00:01, 182.18it/s, Materializing param=model.layers.7.mlp.up_proj.weight] Loading weights: 22%|██▏ | 69/311 [00:00<00:02, 120.27it/s, Materializing param=model.layers.6.input_layernorm.weight] Loading weights: 34%|███▍ | 106/311 [00:00<00:00, 259.35it/s, Materializing param=model.layers.9.post_attention_layernorm.weight] Loading weights: 23%|██▎ | 70/311 [00:00<00:02, 120.27it/s, Materializing param=model.layers.6.mlp.down_proj.weight] Loading weights: 23%|██▎ | 70/311 [00:00<00:02, 120.27it/s, Materializing param=model.layers.6.mlp.down_proj.weight] Loading weights: 34%|███▍ | 107/311 [00:00<00:00, 259.35it/s, Materializing param=model.layers.9.self_attn.k_norm.weight] Loading weights: 34%|███▍ | 107/311 [00:00<00:00, 259.35it/s, Materializing param=model.layers.9.self_attn.k_norm.weight] Loading weights: 35%|███▍ | 108/311 [00:00<00:00, 259.35it/s, Materializing param=model.layers.9.self_attn.k_proj.weight] Loading weights: 35%|███▍ | 108/311 [00:00<00:00, 259.35it/s, Materializing param=model.layers.9.self_attn.k_proj.weight] Loading weights: 35%|███▌ | 109/311 [00:00<00:00, 259.35it/s, Materializing param=model.layers.9.self_attn.o_proj.weight] Loading weights: 37%|███▋ | 115/311 [00:00<00:00, 228.68it/s, Materializing param=model.layers.10.mlp.gate_proj.weight] Loading weights: 35%|███▌ | 109/311 [00:00<00:00, 259.35it/s, Materializing param=model.layers.9.self_attn.o_proj.weight] Loading weights: 37%|███▋ | 115/311 [00:00<00:00, 228.68it/s, Materializing param=model.layers.10.mlp.gate_proj.weight] Loading weights: 41%|████ | 127/311 [00:00<00:00, 297.27it/s, Materializing param=model.layers.11.mlp.up_proj.weight] Loading weights: 41%|████ | 127/311 [00:00<00:00, 254.36it/s, Materializing param=model.layers.11.mlp.up_proj.weight] Loading weights: 41%|████ | 127/311 [00:00<00:00, 297.27it/s, Materializing param=model.layers.11.mlp.up_proj.weight] Loading weights: 41%|████ | 127/311 [00:00<00:00, 254.36it/s, Materializing param=model.layers.11.mlp.up_proj.weight] Loading weights: 35%|███▌ | 110/311 [00:00<00:00, 259.35it/s, Materializing param=model.layers.9.self_attn.q_norm.weight] Loading weights: 35%|███▌ | 110/311 [00:00<00:00, 259.35it/s, Materializing param=model.layers.9.self_attn.q_norm.weight] Loading weights: 36%|███▌ | 111/311 [00:00<00:00, 259.35it/s, Materializing param=model.layers.9.self_attn.q_proj.weight] Loading weights: 36%|███▌ | 111/311 [00:00<00:00, 259.35it/s, Materializing param=model.layers.9.self_attn.q_proj.weight] Loading weights: 13%|█▎ | 39/311 [00:00<00:02, 121.83it/s, Materializing param=model.layers.3.mlp.gate_proj.weight] Loading weights: 13%|█▎ | 39/311 [00:00<00:02, 121.83it/s, Materializing param=model.layers.3.mlp.up_proj.weight] Loading weights: 13%|█▎ | 39/311 [00:00<00:02, 121.83it/s, Materializing param=model.layers.3.mlp.up_proj.weight] Loading weights: 36%|███▌ | 112/311 [00:00<00:00, 259.35it/s, Materializing param=model.layers.9.self_attn.v_proj.weight] Loading weights: 36%|███▌ | 112/311 [00:00<00:00, 259.35it/s, Materializing param=model.layers.9.self_attn.v_proj.weight] Loading weights: 36%|███▋ | 113/311 [00:00<00:00, 259.35it/s, Materializing param=model.layers.10.input_layernorm.weight] Loading weights: 36%|███▋ | 113/311 [00:00<00:00, 259.35it/s, Materializing param=model.layers.10.input_layernorm.weight] Loading weights: 37%|███▋ | 114/311 [00:00<00:00, 259.35it/s, Materializing param=model.layers.10.mlp.down_proj.weight] Loading weights: 37%|███▋ | 114/311 [00:00<00:00, 259.35it/s, Materializing param=model.layers.10.mlp.down_proj.weight] Loading weights: 41%|████ | 128/311 [00:00<00:00, 254.36it/s, Materializing param=model.layers.11.post_attention_layernorm.weight] Loading weights: 41%|████ | 128/311 [00:00<00:00, 297.27it/s, Materializing param=model.layers.11.post_attention_layernorm.weight] Loading weights: 41%|████ | 128/311 [00:00<00:00, 254.36it/s, Materializing param=model.layers.11.post_attention_layernorm.weight] Loading weights: 23%|██▎ | 71/311 [00:00<00:01, 120.27it/s, Materializing param=model.layers.6.mlp.gate_proj.weight] Loading weights: 41%|████ | 128/311 [00:00<00:00, 297.27it/s, Materializing param=model.layers.11.post_attention_layernorm.weight] Loading weights: 23%|██▎ | 71/311 [00:00<00:01, 120.27it/s, Materializing param=model.layers.6.mlp.gate_proj.weight] Loading weights: 41%|████▏ | 129/311 [00:00<00:00, 254.36it/s, Materializing param=model.layers.11.self_attn.k_norm.weight] Loading weights: 41%|████▏ | 129/311 [00:00<00:00, 254.36it/s, Materializing param=model.layers.11.self_attn.k_norm.weight] Loading weights: 41%|████▏ | 129/311 [00:00<00:00, 297.27it/s, Materializing param=model.layers.11.self_attn.k_norm.weight] Loading weights: 27%|██▋ | 84/311 [00:00<00:01, 182.18it/s, Materializing param=model.layers.7.post_attention_layernorm.weight] Loading weights: 41%|████▏ | 129/311 [00:00<00:00, 297.27it/s, Materializing param=model.layers.11.self_attn.k_norm.weight] Loading weights: 27%|██▋ | 84/311 [00:00<00:01, 182.18it/s, Materializing param=model.layers.7.post_attention_layernorm.weight] Loading weights: 42%|████▏ | 130/311 [00:00<00:00, 254.36it/s, Materializing param=model.layers.11.self_attn.k_proj.weight] Loading weights: 42%|████▏ | 130/311 [00:00<00:00, 254.36it/s, Materializing param=model.layers.11.self_attn.k_proj.weight] Loading weights: 27%|██▋ | 85/311 [00:00<00:01, 182.18it/s, Materializing param=model.layers.7.self_attn.k_norm.weight] Loading weights: 27%|██▋ | 85/311 [00:00<00:01, 182.18it/s, Materializing param=model.layers.7.self_attn.k_norm.weight] Loading weights: 42%|████▏ | 130/311 [00:00<00:00, 297.27it/s, Materializing param=model.layers.11.self_attn.k_proj.weight] Loading weights: 28%|██▊ | 86/311 [00:00<00:01, 182.18it/s, Materializing param=model.layers.7.self_attn.k_proj.weight] Loading weights: 42%|████▏ | 130/311 [00:00<00:00, 297.27it/s, Materializing param=model.layers.11.self_attn.k_proj.weight] Loading weights: 28%|██▊ | 86/311 [00:00<00:01, 182.18it/s, Materializing param=model.layers.7.self_attn.k_proj.weight] Loading weights: 37%|███▋ | 116/311 [00:00<00:00, 228.68it/s, Materializing param=model.layers.10.mlp.up_proj.weight] Loading weights: 37%|███▋ | 116/311 [00:00<00:00, 228.68it/s, Materializing param=model.layers.10.mlp.up_proj.weight] Loading weights: 42%|████▏ | 131/311 [00:00<00:00, 297.27it/s, Materializing param=model.layers.11.self_attn.o_proj.weight] Loading weights: 42%|████▏ | 131/311 [00:00<00:00, 297.27it/s, Materializing param=model.layers.11.self_attn.o_proj.weight] Loading weights: 28%|██▊ | 87/311 [00:00<00:01, 182.18it/s, Materializing param=model.layers.7.self_attn.o_proj.weight] Loading weights: 28%|██▊ | 87/311 [00:00<00:01, 182.18it/s, Materializing param=model.layers.7.self_attn.o_proj.weight] Loading weights: 42%|████▏ | 131/311 [00:00<00:00, 254.36it/s, Materializing param=model.layers.11.self_attn.o_proj.weight] Loading weights: 42%|████▏ | 131/311 [00:00<00:00, 254.36it/s, Materializing param=model.layers.11.self_attn.o_proj.weight] Loading weights: 42%|████▏ | 132/311 [00:00<00:00, 254.36it/s, Materializing param=model.layers.11.self_attn.q_norm.weight] Loading weights: 42%|████▏ | 132/311 [00:00<00:00, 254.36it/s, Materializing param=model.layers.11.self_attn.q_norm.weight] Loading weights: 43%|████▎ | 133/311 [00:00<00:00, 254.36it/s, Materializing param=model.layers.11.self_attn.q_proj.weight] Loading weights: 43%|████▎ | 133/311 [00:00<00:00, 254.36it/s, Materializing param=model.layers.11.self_attn.q_proj.weight] Loading weights: 42%|████▏ | 132/311 [00:00<00:00, 297.27it/s, Materializing param=model.layers.11.self_attn.q_norm.weight] Loading weights: 42%|████▏ | 132/311 [00:00<00:00, 297.27it/s, Materializing param=model.layers.11.self_attn.q_norm.weight] Loading weights: 43%|████▎ | 133/311 [00:00<00:00, 297.27it/s, Materializing param=model.layers.11.self_attn.q_proj.weight] Loading weights: 43%|████▎ | 133/311 [00:00<00:00, 297.27it/s, Materializing param=model.layers.11.self_attn.q_proj.weight] Loading weights: 13%|█▎ | 40/311 [00:00<00:02, 121.83it/s, Materializing param=model.layers.3.post_attention_layernorm.weight] Loading weights: 28%|██▊ | 88/311 [00:00<00:01, 182.18it/s, Materializing param=model.layers.7.self_attn.q_norm.weight] Loading weights: 28%|██▊ | 88/311 [00:00<00:01, 182.18it/s, Materializing param=model.layers.7.self_attn.q_norm.weight] Loading weights: 13%|█▎ | 40/311 [00:00<00:02, 121.83it/s, Materializing param=model.layers.3.post_attention_layernorm.weight] Loading weights: 29%|██▊ | 89/311 [00:00<00:01, 182.18it/s, Materializing param=model.layers.7.self_attn.q_proj.weight] Loading weights: 29%|██▊ | 89/311 [00:00<00:01, 182.18it/s, Materializing param=model.layers.7.self_attn.q_proj.weight] Loading weights: 13%|█▎ | 41/311 [00:00<00:02, 121.83it/s, Materializing param=model.layers.3.self_attn.k_norm.weight] Loading weights: 13%|█▎ | 41/311 [00:00<00:02, 121.83it/s, Materializing param=model.layers.3.self_attn.k_norm.weight] Loading weights: 14%|█▎ | 42/311 [00:00<00:02, 121.83it/s, Materializing param=model.layers.3.self_attn.k_proj.weight] Loading weights: 14%|█▎ | 42/311 [00:00<00:02, 121.83it/s, Materializing param=model.layers.3.self_attn.k_proj.weight] Loading weights: 37%|███▋ | 115/311 [00:00<00:00, 259.35it/s, Materializing param=model.layers.10.mlp.gate_proj.weight] Loading weights: 37%|███▋ | 115/311 [00:00<00:00, 259.35it/s, Materializing param=model.layers.10.mlp.gate_proj.weight] Loading weights: 43%|████▎ | 134/311 [00:00<00:00, 254.36it/s, Materializing param=model.layers.11.self_attn.v_proj.weight] Loading weights: 43%|████▎ | 134/311 [00:00<00:00, 297.27it/s, Materializing param=model.layers.11.self_attn.v_proj.weight] Loading weights: 43%|████▎ | 134/311 [00:00<00:00, 254.36it/s, Materializing param=model.layers.11.self_attn.v_proj.weight] Loading weights: 43%|████▎ | 134/311 [00:00<00:00, 297.27it/s, Materializing param=model.layers.11.self_attn.v_proj.weight] Loading weights: 14%|█▍ | 43/311 [00:00<00:02, 121.83it/s, Materializing param=model.layers.3.self_attn.o_proj.weight] Loading weights: 14%|█▍ | 43/311 [00:00<00:02, 121.83it/s, Materializing param=model.layers.3.self_attn.o_proj.weight] Loading weights: 43%|████▎ | 135/311 [00:00<00:00, 254.36it/s, Materializing param=model.layers.12.input_layernorm.weight] Loading weights: 43%|████▎ | 135/311 [00:00<00:00, 254.36it/s, Materializing param=model.layers.12.input_layernorm.weight] Loading weights: 43%|████▎ | 135/311 [00:00<00:00, 297.27it/s, Materializing param=model.layers.12.input_layernorm.weight] Loading weights: 43%|████▎ | 135/311 [00:00<00:00, 297.27it/s, Materializing param=model.layers.12.input_layernorm.weight] Loading weights: 29%|██▉ | 90/311 [00:00<00:01, 182.18it/s, Materializing param=model.layers.7.self_attn.v_proj.weight] Loading weights: 44%|████▎ | 136/311 [00:00<00:00, 297.27it/s, Materializing param=model.layers.12.mlp.down_proj.weight] Loading weights: 29%|██▉ | 90/311 [00:00<00:01, 182.18it/s, Materializing param=model.layers.7.self_attn.v_proj.weight] Loading weights: 44%|████▎ | 136/311 [00:00<00:00, 297.27it/s, Materializing param=model.layers.12.mlp.down_proj.weight] Loading weights: 23%|██▎ | 72/311 [00:00<00:01, 120.27it/s, Materializing param=model.layers.6.mlp.up_proj.weight] Loading weights: 23%|██▎ | 72/311 [00:00<00:01, 120.27it/s, Materializing param=model.layers.6.mlp.up_proj.weight] Loading weights: 29%|██▉ | 91/311 [00:00<00:01, 182.18it/s, Materializing param=model.layers.8.input_layernorm.weight] Loading weights: 29%|██▉ | 91/311 [00:00<00:01, 182.18it/s, Materializing param=model.layers.8.input_layernorm.weight] Loading weights: 38%|███▊ | 117/311 [00:00<00:00, 228.68it/s, Materializing param=model.layers.10.post_attention_layernorm.weight] Loading weights: 38%|███▊ | 117/311 [00:00<00:00, 228.68it/s, Materializing param=model.layers.10.post_attention_layernorm.weight] Loading weights: 30%|██▉ | 92/311 [00:00<00:01, 182.18it/s, Materializing param=model.layers.8.mlp.down_proj.weight] Loading weights: 30%|██▉ | 92/311 [00:00<00:01, 182.18it/s, Materializing param=model.layers.8.mlp.down_proj.weight] Loading weights: 38%|███▊ | 118/311 [00:00<00:00, 228.68it/s, Materializing param=model.layers.10.self_attn.k_norm.weight] Loading weights: 38%|███▊ | 118/311 [00:00<00:00, 228.68it/s, Materializing param=model.layers.10.self_attn.k_norm.weight] Loading weights: 38%|███▊ | 119/311 [00:00<00:00, 228.68it/s, Materializing param=model.layers.10.self_attn.k_proj.weight] Loading weights: 38%|███▊ | 119/311 [00:00<00:00, 228.68it/s, Materializing param=model.layers.10.self_attn.k_proj.weight] Loading weights: 39%|███▊ | 120/311 [00:00<00:00, 228.68it/s, Materializing param=model.layers.10.self_attn.o_proj.weight] Loading weights: 39%|███▊ | 120/311 [00:00<00:00, 228.68it/s, Materializing param=model.layers.10.self_attn.o_proj.weight] Loading weights: 14%|█▍ | 44/311 [00:00<00:02, 121.83it/s, Materializing param=model.layers.3.self_attn.q_norm.weight] Loading weights: 14%|█▍ | 44/311 [00:00<00:02, 121.83it/s, Materializing param=model.layers.3.self_attn.q_norm.weight] Loading weights: 14%|█▍ | 45/311 [00:00<00:02, 121.83it/s, Materializing param=model.layers.3.self_attn.q_proj.weight] Loading weights: 14%|█▍ | 45/311 [00:00<00:02, 121.83it/s, Materializing param=model.layers.3.self_attn.q_proj.weight] Loading weights: 15%|█▍ | 46/311 [00:00<00:02, 121.83it/s, Materializing param=model.layers.3.self_attn.v_proj.weight] Loading weights: 15%|█▍ | 46/311 [00:00<00:02, 121.83it/s, Materializing param=model.layers.3.self_attn.v_proj.weight] Loading weights: 39%|███▉ | 121/311 [00:00<00:00, 228.68it/s, Materializing param=model.layers.10.self_attn.q_norm.weight] Loading weights: 39%|███▉ | 121/311 [00:00<00:00, 228.68it/s, Materializing param=model.layers.10.self_attn.q_norm.weight] Loading weights: 15%|█▌ | 47/311 [00:00<00:02, 121.83it/s, Materializing param=model.layers.4.input_layernorm.weight] Loading weights: 39%|███▉ | 122/311 [00:00<00:00, 228.68it/s, Materializing param=model.layers.10.self_attn.q_proj.weight] Loading weights: 15%|█▌ | 47/311 [00:00<00:02, 121.83it/s, Materializing param=model.layers.4.input_layernorm.weight] Loading weights: 39%|███▉ | 122/311 [00:00<00:00, 228.68it/s, Materializing param=model.layers.10.self_attn.q_proj.weight] Loading weights: 15%|█▌ | 48/311 [00:00<00:02, 121.83it/s, Materializing param=model.layers.4.mlp.down_proj.weight] Loading weights: 15%|█▌ | 48/311 [00:00<00:02, 121.83it/s, Materializing param=model.layers.4.mlp.down_proj.weight] Loading weights: 37%|███▋ | 116/311 [00:00<00:00, 259.35it/s, Materializing param=model.layers.10.mlp.up_proj.weight] Loading weights: 37%|███▋ | 116/311 [00:00<00:00, 259.35it/s, Materializing param=model.layers.10.mlp.up_proj.weight] Loading weights: 40%|███▉ | 123/311 [00:00<00:00, 228.68it/s, Materializing param=model.layers.10.self_attn.v_proj.weight] Loading weights: 40%|███▉ | 123/311 [00:00<00:00, 228.68it/s, Materializing param=model.layers.10.self_attn.v_proj.weight] Loading weights: 44%|████▎ | 136/311 [00:00<00:00, 254.36it/s, Materializing param=model.layers.12.mlp.down_proj.weight] Loading weights: 44%|████▍ | 137/311 [00:00<00:00, 297.27it/s, Materializing param=model.layers.12.mlp.gate_proj.weight] Loading weights: 44%|████▎ | 136/311 [00:00<00:00, 254.36it/s, Materializing param=model.layers.12.mlp.down_proj.weight] Loading weights: 44%|████▍ | 137/311 [00:00<00:00, 297.27it/s, Materializing param=model.layers.12.mlp.gate_proj.weight] Loading weights: 44%|████▍ | 137/311 [00:00<00:00, 254.36it/s, Materializing param=model.layers.12.mlp.gate_proj.weight] Loading weights: 44%|████▍ | 137/311 [00:00<00:00, 254.36it/s, Materializing param=model.layers.12.mlp.gate_proj.weight] Loading weights: 40%|███▉ | 124/311 [00:00<00:00, 228.68it/s, Materializing param=model.layers.11.input_layernorm.weight] Loading weights: 40%|███▉ | 124/311 [00:00<00:00, 228.68it/s, Materializing param=model.layers.11.input_layernorm.weight] Loading weights: 40%|████ | 125/311 [00:00<00:00, 228.68it/s, Materializing param=model.layers.11.mlp.down_proj.weight] Loading weights: 40%|████ | 125/311 [00:00<00:00, 228.68it/s, Materializing param=model.layers.11.mlp.down_proj.weight] Loading weights: 23%|██▎ | 73/311 [00:00<00:01, 120.27it/s, Materializing param=model.layers.6.post_attention_layernorm.weight] Loading weights: 23%|██▎ | 73/311 [00:00<00:01, 120.27it/s, Materializing param=model.layers.6.post_attention_layernorm.weight] Loading weights: 24%|██▍ | 74/311 [00:00<00:01, 120.27it/s, Materializing param=model.layers.6.self_attn.k_norm.weight] Loading weights: 24%|██▍ | 74/311 [00:00<00:01, 120.27it/s, Materializing param=model.layers.6.self_attn.k_norm.weight] Loading weights: 24%|██▍ | 75/311 [00:00<00:01, 120.27it/s, Materializing param=model.layers.6.self_attn.k_proj.weight] Loading weights: 24%|██▍ | 75/311 [00:00<00:01, 120.27it/s, Materializing param=model.layers.6.self_attn.k_proj.weight] Loading weights: 30%|██▉ | 93/311 [00:00<00:01, 182.18it/s, Materializing param=model.layers.8.mlp.gate_proj.weight] Loading weights: 30%|██▉ | 93/311 [00:00<00:01, 182.18it/s, Materializing param=model.layers.8.mlp.gate_proj.weight] Loading weights: 24%|██▍ | 76/311 [00:00<00:01, 120.27it/s, Materializing param=model.layers.6.self_attn.o_proj.weight] Loading weights: 24%|██▍ | 76/311 [00:00<00:01, 120.27it/s, Materializing param=model.layers.6.self_attn.o_proj.weight] Loading weights: 16%|█▌ | 49/311 [00:00<00:02, 121.83it/s, Materializing param=model.layers.4.mlp.gate_proj.weight] Loading weights: 16%|█▌ | 49/311 [00:00<00:02, 121.83it/s, Materializing param=model.layers.4.mlp.gate_proj.weight] Loading weights: 38%|███▊ | 117/311 [00:00<00:00, 259.35it/s, Materializing param=model.layers.10.post_attention_layernorm.weight] Loading weights: 38%|███▊ | 117/311 [00:00<00:00, 259.35it/s, Materializing param=model.layers.10.post_attention_layernorm.weight] Loading weights: 38%|███▊ | 118/311 [00:00<00:00, 259.35it/s, Materializing param=model.layers.10.self_attn.k_norm.weight] Loading weights: 38%|███▊ | 118/311 [00:00<00:00, 259.35it/s, Materializing param=model.layers.10.self_attn.k_norm.weight] Loading weights: 38%|███▊ | 119/311 [00:00<00:00, 259.35it/s, Materializing param=model.layers.10.self_attn.k_proj.weight] Loading weights: 38%|███▊ | 119/311 [00:00<00:00, 259.35it/s, Materializing param=model.layers.10.self_attn.k_proj.weight] Loading weights: 44%|████▍ | 138/311 [00:00<00:00, 313.51it/s, Materializing param=model.layers.12.mlp.gate_proj.weight] Loading weights: 44%|████▍ | 138/311 [00:00<00:00, 313.51it/s, Materializing param=model.layers.12.mlp.up_proj.weight] Loading weights: 41%|████ | 126/311 [00:00<00:00, 228.68it/s, Materializing param=model.layers.11.mlp.gate_proj.weight] Loading weights: 44%|████▍ | 138/311 [00:00<00:00, 297.27it/s, Materializing param=model.layers.12.mlp.up_proj.weight] Loading weights: 44%|████▍ | 138/311 [00:00<00:00, 313.51it/s, Materializing param=model.layers.12.mlp.up_proj.weight] Loading weights: 44%|████▍ | 138/311 [00:00<00:00, 297.27it/s, Materializing param=model.layers.12.mlp.up_proj.weight] Loading weights: 41%|████ | 126/311 [00:00<00:00, 228.68it/s, Materializing param=model.layers.11.mlp.gate_proj.weight] Loading weights: 39%|███▊ | 120/311 [00:00<00:00, 259.35it/s, Materializing param=model.layers.10.self_attn.o_proj.weight] Loading weights: 25%|██▍ | 77/311 [00:00<00:01, 200.31it/s, Materializing param=model.layers.6.self_attn.o_proj.weight] Loading weights: 25%|██▍ | 77/311 [00:00<00:01, 200.31it/s, Materializing param=model.layers.6.self_attn.q_norm.weight] Loading weights: 39%|███▊ | 120/311 [00:00<00:00, 259.35it/s, Materializing param=model.layers.10.self_attn.o_proj.weight] Loading weights: 25%|██▍ | 77/311 [00:00<00:01, 200.31it/s, Materializing param=model.layers.6.self_attn.q_norm.weight] Loading weights: 25%|██▌ | 78/311 [00:00<00:01, 200.31it/s, Materializing param=model.layers.6.self_attn.q_proj.weight] Loading weights: 39%|███▉ | 121/311 [00:00<00:00, 259.35it/s, Materializing param=model.layers.10.self_attn.q_norm.weight] Loading weights: 25%|██▌ | 78/311 [00:00<00:01, 200.31it/s, Materializing param=model.layers.6.self_attn.q_proj.weight] Loading weights: 39%|███▉ | 121/311 [00:00<00:00, 259.35it/s, Materializing param=model.layers.10.self_attn.q_norm.weight] Loading weights: 25%|██▌ | 79/311 [00:00<00:01, 200.31it/s, Materializing param=model.layers.6.self_attn.v_proj.weight] Loading weights: 25%|██▌ | 79/311 [00:00<00:01, 200.31it/s, Materializing param=model.layers.6.self_attn.v_proj.weight] Loading weights: 39%|███▉ | 122/311 [00:00<00:00, 259.35it/s, Materializing param=model.layers.10.self_attn.q_proj.weight] Loading weights: 39%|███▉ | 122/311 [00:00<00:00, 259.35it/s, Materializing param=model.layers.10.self_attn.q_proj.weight] Loading weights: 26%|██▌ | 80/311 [00:00<00:01, 200.31it/s, Materializing param=model.layers.7.input_layernorm.weight] Loading weights: 30%|███ | 94/311 [00:00<00:01, 182.18it/s, Materializing param=model.layers.8.mlp.up_proj.weight] Loading weights: 26%|██▌ | 80/311 [00:00<00:01, 200.31it/s, Materializing param=model.layers.7.input_layernorm.weight] Loading weights: 30%|███ | 94/311 [00:00<00:01, 182.18it/s, Materializing param=model.layers.8.mlp.up_proj.weight] Loading weights: 40%|███▉ | 123/311 [00:00<00:00, 259.35it/s, Materializing param=model.layers.10.self_attn.v_proj.weight] Loading weights: 40%|███▉ | 123/311 [00:00<00:00, 259.35it/s, Materializing param=model.layers.10.self_attn.v_proj.weight] Loading weights: 40%|███▉ | 124/311 [00:00<00:00, 259.35it/s, Materializing param=model.layers.11.input_layernorm.weight] Loading weights: 40%|███▉ | 124/311 [00:00<00:00, 259.35it/s, Materializing param=model.layers.11.input_layernorm.weight] Loading weights: 40%|████ | 125/311 [00:00<00:00, 259.35it/s, Materializing param=model.layers.11.mlp.down_proj.weight] Loading weights: 40%|████ | 125/311 [00:00<00:00, 259.35it/s, Materializing param=model.layers.11.mlp.down_proj.weight] Loading weights: 16%|█▌ | 50/311 [00:00<00:02, 121.83it/s, Materializing param=model.layers.4.mlp.up_proj.weight] Loading weights: 16%|█▌ | 50/311 [00:00<00:02, 121.83it/s, Materializing param=model.layers.4.mlp.up_proj.weight] Loading weights: 45%|████▍ | 139/311 [00:00<00:00, 313.51it/s, Materializing param=model.layers.12.post_attention_layernorm.weight] Loading weights: 45%|████▍ | 139/311 [00:00<00:00, 313.51it/s, Materializing param=model.layers.12.post_attention_layernorm.weight] Loading weights: 45%|████▍ | 139/311 [00:00<00:00, 297.27it/s, Materializing param=model.layers.12.post_attention_layernorm.weight] Loading weights: 45%|████▍ | 139/311 [00:00<00:00, 297.27it/s, Materializing param=model.layers.12.post_attention_layernorm.weight] Loading weights: 45%|████▌ | 140/311 [00:00<00:00, 313.51it/s, Materializing param=model.layers.12.self_attn.k_norm.weight] Loading weights: 45%|████▌ | 140/311 [00:00<00:00, 313.51it/s, Materializing param=model.layers.12.self_attn.k_norm.weight] Loading weights: 45%|████▌ | 140/311 [00:00<00:00, 297.27it/s, Materializing param=model.layers.12.self_attn.k_norm.weight] Loading weights: 45%|████▌ | 140/311 [00:00<00:00, 297.27it/s, Materializing param=model.layers.12.self_attn.k_norm.weight] Loading weights: 45%|████▌ | 141/311 [00:00<00:00, 297.27it/s, Materializing param=model.layers.12.self_attn.k_proj.weight] Loading weights: 45%|████▌ | 141/311 [00:00<00:00, 313.51it/s, Materializing param=model.layers.12.self_attn.k_proj.weight] Loading weights: 45%|████▌ | 141/311 [00:00<00:00, 297.27it/s, Materializing param=model.layers.12.self_attn.k_proj.weight] Loading weights: 45%|████▌ | 141/311 [00:00<00:00, 313.51it/s, Materializing param=model.layers.12.self_attn.k_proj.weight] Loading weights: 41%|████ | 127/311 [00:00<00:00, 228.68it/s, Materializing param=model.layers.11.mlp.up_proj.weight] Loading weights: 41%|████ | 127/311 [00:00<00:00, 228.68it/s, Materializing param=model.layers.11.mlp.up_proj.weight] Loading weights: 46%|████▌ | 142/311 [00:00<00:00, 313.51it/s, Materializing param=model.layers.12.self_attn.o_proj.weight] Loading weights: 46%|████▌ | 142/311 [00:00<00:00, 313.51it/s, Materializing param=model.layers.12.self_attn.o_proj.weight] Loading weights: 46%|████▌ | 142/311 [00:00<00:00, 297.27it/s, Materializing param=model.layers.12.self_attn.o_proj.weight] Loading weights: 46%|████▌ | 142/311 [00:00<00:00, 297.27it/s, Materializing param=model.layers.12.self_attn.o_proj.weight] Loading weights: 46%|████▌ | 143/311 [00:00<00:00, 297.27it/s, Materializing param=model.layers.12.self_attn.q_norm.weight] Loading weights: 26%|██▌ | 81/311 [00:00<00:01, 200.31it/s, Materializing param=model.layers.7.mlp.down_proj.weight] Loading weights: 46%|████▌ | 143/311 [00:00<00:00, 297.27it/s, Materializing param=model.layers.12.self_attn.q_norm.weight] Loading weights: 26%|██▌ | 81/311 [00:00<00:01, 200.31it/s, Materializing param=model.layers.7.mlp.down_proj.weight] Loading weights: 26%|██▋ | 82/311 [00:00<00:01, 200.31it/s, Materializing param=model.layers.7.mlp.gate_proj.weight] Loading weights: 26%|██▋ | 82/311 [00:00<00:01, 200.31it/s, Materializing param=model.layers.7.mlp.gate_proj.weight] Loading weights: 46%|████▋ | 144/311 [00:00<00:00, 297.27it/s, Materializing param=model.layers.12.self_attn.q_proj.weight] Loading weights: 46%|████▋ | 144/311 [00:00<00:00, 297.27it/s, Materializing param=model.layers.12.self_attn.q_proj.weight] Loading weights: 31%|███ | 95/311 [00:00<00:01, 182.18it/s, Materializing param=model.layers.8.post_attention_layernorm.weight] Loading weights: 31%|███ | 95/311 [00:00<00:01, 182.18it/s, Materializing param=model.layers.8.post_attention_layernorm.weight] Loading weights: 31%|███ | 96/311 [00:00<00:01, 182.18it/s, Materializing param=model.layers.8.self_attn.k_norm.weight] Loading weights: 31%|███ | 96/311 [00:00<00:01, 182.18it/s, Materializing param=model.layers.8.self_attn.k_norm.weight] Loading weights: 31%|███ | 97/311 [00:00<00:01, 182.18it/s, Materializing param=model.layers.8.self_attn.k_proj.weight] Loading weights: 31%|███ | 97/311 [00:00<00:01, 182.18it/s, Materializing param=model.layers.8.self_attn.k_proj.weight] Loading weights: 46%|████▌ | 143/311 [00:00<00:00, 313.51it/s, Materializing param=model.layers.12.self_attn.q_norm.weight] Loading weights: 46%|████▌ | 143/311 [00:00<00:00, 313.51it/s, Materializing param=model.layers.12.self_attn.q_norm.weight] Loading weights: 46%|████▋ | 144/311 [00:00<00:00, 313.51it/s, Materializing param=model.layers.12.self_attn.q_proj.weight] Loading weights: 46%|████▋ | 144/311 [00:00<00:00, 313.51it/s, Materializing param=model.layers.12.self_attn.q_proj.weight] Loading weights: 47%|████▋ | 145/311 [00:00<00:00, 297.27it/s, Materializing param=model.layers.12.self_attn.v_proj.weight] Loading weights: 47%|████▋ | 145/311 [00:00<00:00, 297.27it/s, Materializing param=model.layers.12.self_attn.v_proj.weight] Loading weights: 47%|████▋ | 145/311 [00:00<00:00, 313.51it/s, Materializing param=model.layers.12.self_attn.v_proj.weight] Loading weights: 41%|████ | 126/311 [00:00<00:00, 259.35it/s, Materializing param=model.layers.11.mlp.gate_proj.weight] Loading weights: 47%|████▋ | 145/311 [00:00<00:00, 313.51it/s, Materializing param=model.layers.12.self_attn.v_proj.weight] Loading weights: 32%|███▏ | 98/311 [00:00<00:01, 182.18it/s, Materializing param=model.layers.8.self_attn.o_proj.weight] Loading weights: 41%|████ | 126/311 [00:00<00:00, 259.35it/s, Materializing param=model.layers.11.mlp.gate_proj.weight] Loading weights: 32%|███▏ | 98/311 [00:00<00:01, 182.18it/s, Materializing param=model.layers.8.self_attn.o_proj.weight] Loading weights: 16%|█▋ | 51/311 [00:00<00:02, 121.83it/s, Materializing param=model.layers.4.post_attention_layernorm.weight] Loading weights: 16%|█▋ | 51/311 [00:00<00:02, 121.83it/s, Materializing param=model.layers.4.post_attention_layernorm.weight] Loading weights: 17%|█▋ | 52/311 [00:00<00:02, 121.83it/s, Materializing param=model.layers.4.self_attn.k_norm.weight] Loading weights: 47%|████▋ | 146/311 [00:00<00:00, 297.27it/s, Materializing param=model.layers.13.input_layernorm.weight] Loading weights: 47%|████▋ | 146/311 [00:00<00:00, 313.51it/s, Materializing param=model.layers.13.input_layernorm.weight] Loading weights: 17%|█▋ | 52/311 [00:00<00:02, 121.83it/s, Materializing param=model.layers.4.self_attn.k_norm.weight] Loading weights: 47%|████▋ | 146/311 [00:00<00:00, 313.51it/s, Materializing param=model.layers.13.input_layernorm.weight] Loading weights: 47%|████▋ | 146/311 [00:00<00:00, 297.27it/s, Materializing param=model.layers.13.input_layernorm.weight] Loading weights: 17%|█▋ | 53/311 [00:00<00:02, 121.83it/s, Materializing param=model.layers.4.self_attn.k_proj.weight] Loading weights: 47%|████▋ | 147/311 [00:00<00:00, 313.51it/s, Materializing param=model.layers.13.mlp.down_proj.weight] Loading weights: 47%|████▋ | 147/311 [00:00<00:00, 297.27it/s, Materializing param=model.layers.13.mlp.down_proj.weight] Loading weights: 17%|█▋ | 53/311 [00:00<00:02, 121.83it/s, Materializing param=model.layers.4.self_attn.k_proj.weight] Loading weights: 47%|████▋ | 147/311 [00:00<00:00, 313.51it/s, Materializing param=model.layers.13.mlp.down_proj.weight] Loading weights: 47%|████▋ | 147/311 [00:00<00:00, 297.27it/s, Materializing param=model.layers.13.mlp.down_proj.weight] Loading weights: 17%|█▋ | 54/311 [00:00<00:02, 121.83it/s, Materializing param=model.layers.4.self_attn.o_proj.weight] Loading weights: 17%|█▋ | 54/311 [00:00<00:02, 121.83it/s, Materializing param=model.layers.4.self_attn.o_proj.weight] Loading weights: 41%|████ | 128/311 [00:00<00:00, 290.52it/s, Materializing param=model.layers.11.mlp.up_proj.weight] Loading weights: 32%|███▏ | 99/311 [00:00<00:01, 182.18it/s, Materializing param=model.layers.8.self_attn.q_norm.weight] Loading weights: 41%|████ | 128/311 [00:00<00:00, 290.52it/s, Materializing param=model.layers.11.post_attention_layernorm.weight] Loading weights: 32%|███▏ | 99/311 [00:00<00:01, 182.18it/s, Materializing param=model.layers.8.self_attn.q_norm.weight] Loading weights: 41%|████ | 128/311 [00:00<00:00, 290.52it/s, Materializing param=model.layers.11.post_attention_layernorm.weight] Loading weights: 41%|████▏ | 129/311 [00:00<00:00, 290.52it/s, Materializing param=model.layers.11.self_attn.k_norm.weight] Loading weights: 41%|████▏ | 129/311 [00:00<00:00, 290.52it/s, Materializing param=model.layers.11.self_attn.k_norm.weight] Loading weights: 42%|████▏ | 130/311 [00:00<00:00, 290.52it/s, Materializing param=model.layers.11.self_attn.k_proj.weight] Loading weights: 42%|████▏ | 130/311 [00:00<00:00, 290.52it/s, Materializing param=model.layers.11.self_attn.k_proj.weight] Loading weights: 42%|████▏ | 131/311 [00:00<00:00, 290.52it/s, Materializing param=model.layers.11.self_attn.o_proj.weight] Loading weights: 42%|████▏ | 131/311 [00:00<00:00, 290.52it/s, Materializing param=model.layers.11.self_attn.o_proj.weight] Loading weights: 18%|█▊ | 55/311 [00:00<00:02, 121.83it/s, Materializing param=model.layers.4.self_attn.q_norm.weight] Loading weights: 18%|█▊ | 55/311 [00:00<00:02, 121.83it/s, Materializing param=model.layers.4.self_attn.q_norm.weight] Loading weights: 18%|█▊ | 56/311 [00:00<00:02, 121.83it/s, Materializing param=model.layers.4.self_attn.q_proj.weight] Loading weights: 18%|█▊ | 56/311 [00:00<00:02, 121.83it/s, Materializing param=model.layers.4.self_attn.q_proj.weight] Loading weights: 32%|███▏ | 100/311 [00:00<00:01, 182.18it/s, Materializing param=model.layers.8.self_attn.q_proj.weight] Loading weights: 32%|███▏ | 100/311 [00:00<00:01, 182.18it/s, Materializing param=model.layers.8.self_attn.q_proj.weight] Loading weights: 32%|███▏ | 101/311 [00:00<00:01, 182.18it/s, Materializing param=model.layers.8.self_attn.v_proj.weight] Loading weights: 27%|██▋ | 83/311 [00:00<00:01, 200.31it/s, Materializing param=model.layers.7.mlp.up_proj.weight] Loading weights: 32%|███▏ | 101/311 [00:00<00:01, 182.18it/s, Materializing param=model.layers.8.self_attn.v_proj.weight] Loading weights: 42%|████▏ | 132/311 [00:00<00:00, 290.52it/s, Materializing param=model.layers.11.self_attn.q_norm.weight] Loading weights: 27%|██▋ | 83/311 [00:00<00:01, 200.31it/s, Materializing param=model.layers.7.mlp.up_proj.weight] Loading weights: 42%|████▏ | 132/311 [00:00<00:00, 290.52it/s, Materializing param=model.layers.11.self_attn.q_norm.weight] Loading weights: 43%|████▎ | 133/311 [00:00<00:00, 290.52it/s, Materializing param=model.layers.11.self_attn.q_proj.weight] Loading weights: 41%|████ | 127/311 [00:00<00:00, 259.35it/s, Materializing param=model.layers.11.mlp.up_proj.weight] Loading weights: 43%|████▎ | 133/311 [00:00<00:00, 290.52it/s, Materializing param=model.layers.11.self_attn.q_proj.weight] Loading weights: 41%|████ | 127/311 [00:00<00:00, 259.35it/s, Materializing param=model.layers.11.mlp.up_proj.weight] Loading weights: 33%|███▎ | 102/311 [00:00<00:01, 182.18it/s, Materializing param=model.layers.9.input_layernorm.weight] Loading weights: 48%|████▊ | 148/311 [00:00<00:00, 313.51it/s, Materializing param=model.layers.13.mlp.gate_proj.weight] Loading weights: 33%|███▎ | 102/311 [00:00<00:01, 182.18it/s, Materializing param=model.layers.9.input_layernorm.weight] Loading weights: 48%|████▊ | 148/311 [00:00<00:00, 313.51it/s, Materializing param=model.layers.13.mlp.gate_proj.weight] Loading weights: 48%|████▊ | 148/311 [00:00<00:00, 297.27it/s, Materializing param=model.layers.13.mlp.gate_proj.weight] Loading weights: 18%|█▊ | 57/311 [00:00<00:02, 121.83it/s, Materializing param=model.layers.4.self_attn.v_proj.weight] Loading weights: 48%|████▊ | 148/311 [00:00<00:00, 297.27it/s, Materializing param=model.layers.13.mlp.gate_proj.weight] Loading weights: 18%|█▊ | 57/311 [00:00<00:02, 121.83it/s, Materializing param=model.layers.4.self_attn.v_proj.weight] Loading weights: 33%|███▎ | 103/311 [00:00<00:01, 182.18it/s, Materializing param=model.layers.9.mlp.down_proj.weight] Loading weights: 33%|███▎ | 103/311 [00:00<00:01, 182.18it/s, Materializing param=model.layers.9.mlp.down_proj.weight] Loading weights: 43%|████▎ | 134/311 [00:00<00:00, 290.52it/s, Materializing param=model.layers.11.self_attn.v_proj.weight] Loading weights: 19%|█▊ | 58/311 [00:00<00:02, 121.83it/s, Materializing param=model.layers.5.input_layernorm.weight] Loading weights: 43%|████▎ | 134/311 [00:00<00:00, 290.52it/s, Materializing param=model.layers.11.self_attn.v_proj.weight] Loading weights: 19%|█▊ | 58/311 [00:00<00:02, 121.83it/s, Materializing param=model.layers.5.input_layernorm.weight] Loading weights: 43%|████▎ | 135/311 [00:00<00:00, 290.52it/s, Materializing param=model.layers.12.input_layernorm.weight] Loading weights: 43%|████▎ | 135/311 [00:00<00:00, 290.52it/s, Materializing param=model.layers.12.input_layernorm.weight] Loading weights: 44%|████▎ | 136/311 [00:00<00:00, 290.52it/s, Materializing param=model.layers.12.mlp.down_proj.weight] Loading weights: 44%|████▎ | 136/311 [00:00<00:00, 290.52it/s, Materializing param=model.layers.12.mlp.down_proj.weight] Loading weights: 41%|████ | 128/311 [00:00<00:00, 259.35it/s, Materializing param=model.layers.11.post_attention_layernorm.weight] Loading weights: 41%|████ | 128/311 [00:00<00:00, 259.35it/s, Materializing param=model.layers.11.post_attention_layernorm.weight] Loading weights: 41%|████▏ | 129/311 [00:00<00:00, 259.35it/s, Materializing param=model.layers.11.self_attn.k_norm.weight] Loading weights: 41%|████▏ | 129/311 [00:00<00:00, 259.35it/s, Materializing param=model.layers.11.self_attn.k_norm.weight] Loading weights: 42%|████▏ | 130/311 [00:00<00:00, 259.35it/s, Materializing param=model.layers.11.self_attn.k_proj.weight] Loading weights: 42%|████▏ | 130/311 [00:00<00:00, 259.35it/s, Materializing param=model.layers.11.self_attn.k_proj.weight] Loading weights: 27%|██▋ | 84/311 [00:00<00:01, 200.31it/s, Materializing param=model.layers.7.post_attention_layernorm.weight] Loading weights: 42%|████▏ | 131/311 [00:00<00:00, 259.35it/s, Materializing param=model.layers.11.self_attn.o_proj.weight] Loading weights: 27%|██▋ | 84/311 [00:00<00:01, 200.31it/s, Materializing param=model.layers.7.post_attention_layernorm.weight] Loading weights: 42%|████▏ | 131/311 [00:00<00:00, 259.35it/s, Materializing param=model.layers.11.self_attn.o_proj.weight] Loading weights: 27%|██▋ | 85/311 [00:00<00:01, 200.31it/s, Materializing param=model.layers.7.self_attn.k_norm.weight] Loading weights: 48%|████▊ | 149/311 [00:00<00:00, 297.27it/s, Materializing param=model.layers.13.mlp.up_proj.weight] Loading weights: 27%|██▋ | 85/311 [00:00<00:01, 200.31it/s, Materializing param=model.layers.7.self_attn.k_norm.weight] Loading weights: 48%|████▊ | 149/311 [00:00<00:00, 313.51it/s, Materializing param=model.layers.13.mlp.up_proj.weight] Loading weights: 48%|████▊ | 149/311 [00:00<00:00, 297.27it/s, Materializing param=model.layers.13.mlp.up_proj.weight] Loading weights: 48%|████▊ | 149/311 [00:00<00:00, 313.51it/s, Materializing param=model.layers.13.mlp.up_proj.weight] Loading weights: 28%|██▊ | 86/311 [00:00<00:01, 200.31it/s, Materializing param=model.layers.7.self_attn.k_proj.weight] Loading weights: 28%|██▊ | 86/311 [00:00<00:01, 200.31it/s, Materializing param=model.layers.7.self_attn.k_proj.weight] Loading weights: 33%|███▎ | 104/311 [00:00<00:01, 182.18it/s, Materializing param=model.layers.9.mlp.gate_proj.weight] Loading weights: 33%|███▎ | 104/311 [00:00<00:01, 182.18it/s, Materializing param=model.layers.9.mlp.gate_proj.weight] Loading weights: 28%|██▊ | 87/311 [00:00<00:01, 200.31it/s, Materializing param=model.layers.7.self_attn.o_proj.weight] Loading weights: 28%|██▊ | 87/311 [00:00<00:01, 200.31it/s, Materializing param=model.layers.7.self_attn.o_proj.weight] Loading weights: 44%|████▍ | 137/311 [00:00<00:00, 290.52it/s, Materializing param=model.layers.12.mlp.gate_proj.weight] Loading weights: 44%|████▍ | 137/311 [00:00<00:00, 290.52it/s, Materializing param=model.layers.12.mlp.gate_proj.weight] Loading weights: 42%|████▏ | 132/311 [00:00<00:00, 259.35it/s, Materializing param=model.layers.11.self_attn.q_norm.weight] Loading weights: 42%|████▏ | 132/311 [00:00<00:00, 259.35it/s, Materializing param=model.layers.11.self_attn.q_norm.weight] Loading weights: 19%|█▉ | 59/311 [00:00<00:02, 121.83it/s, Materializing param=model.layers.5.mlp.down_proj.weight] Loading weights: 43%|████▎ | 133/311 [00:00<00:00, 259.35it/s, Materializing param=model.layers.11.self_attn.q_proj.weight] Loading weights: 43%|████▎ | 133/311 [00:00<00:00, 259.35it/s, Materializing param=model.layers.11.self_attn.q_proj.weight] Loading weights: 19%|█▉ | 59/311 [00:00<00:02, 121.83it/s, Materializing param=model.layers.5.mlp.down_proj.weight] Loading weights: 19%|█▉ | 60/311 [00:00<00:02, 121.83it/s, Materializing param=model.layers.5.mlp.gate_proj.weight] Loading weights: 19%|█▉ | 60/311 [00:00<00:02, 121.83it/s, Materializing param=model.layers.5.mlp.gate_proj.weight] Loading weights: 28%|██▊ | 88/311 [00:00<00:01, 200.31it/s, Materializing param=model.layers.7.self_attn.q_norm.weight] Loading weights: 28%|██▊ | 88/311 [00:00<00:01, 200.31it/s, Materializing param=model.layers.7.self_attn.q_norm.weight] Loading weights: 43%|████▎ | 134/311 [00:00<00:00, 259.35it/s, Materializing param=model.layers.11.self_attn.v_proj.weight] Loading weights: 43%|████▎ | 134/311 [00:00<00:00, 259.35it/s, Materializing param=model.layers.11.self_attn.v_proj.weight] Loading weights: 43%|████▎ | 135/311 [00:00<00:00, 259.35it/s, Materializing param=model.layers.12.input_layernorm.weight] Loading weights: 43%|████▎ | 135/311 [00:00<00:00, 259.35it/s, Materializing param=model.layers.12.input_layernorm.weight] Loading weights: 44%|████▎ | 136/311 [00:00<00:00, 259.35it/s, Materializing param=model.layers.12.mlp.down_proj.weight] Loading weights: 44%|████▎ | 136/311 [00:00<00:00, 259.35it/s, Materializing param=model.layers.12.mlp.down_proj.weight] Loading weights: 29%|██▊ | 89/311 [00:00<00:01, 200.31it/s, Materializing param=model.layers.7.self_attn.q_proj.weight] Loading weights: 29%|██▊ | 89/311 [00:00<00:01, 200.31it/s, Materializing param=model.layers.7.self_attn.q_proj.weight] Loading weights: 29%|██▉ | 90/311 [00:00<00:01, 200.31it/s, Materializing param=model.layers.7.self_attn.v_proj.weight] Loading weights: 29%|██▉ | 90/311 [00:00<00:01, 200.31it/s, Materializing param=model.layers.7.self_attn.v_proj.weight] Loading weights: 29%|██▉ | 91/311 [00:00<00:01, 200.31it/s, Materializing param=model.layers.8.input_layernorm.weight] Loading weights: 29%|██▉ | 91/311 [00:00<00:01, 200.31it/s, Materializing param=model.layers.8.input_layernorm.weight] Loading weights: 48%|████▊ | 150/311 [00:00<00:00, 313.51it/s, Materializing param=model.layers.13.post_attention_layernorm.weight] Loading weights: 48%|████▊ | 150/311 [00:00<00:00, 313.51it/s, Materializing param=model.layers.13.post_attention_layernorm.weight] Loading weights: 48%|████▊ | 150/311 [00:00<00:00, 345.22it/s, Materializing param=model.layers.13.mlp.up_proj.weight] Loading weights: 30%|██▉ | 92/311 [00:00<00:01, 200.31it/s, Materializing param=model.layers.8.mlp.down_proj.weight] Loading weights: 30%|██▉ | 92/311 [00:00<00:01, 200.31it/s, Materializing param=model.layers.8.mlp.down_proj.weight] Loading weights: 48%|████▊ | 150/311 [00:00<00:00, 345.22it/s, Materializing param=model.layers.13.post_attention_layernorm.weight] Loading weights: 49%|████▊ | 151/311 [00:00<00:00, 313.51it/s, Materializing param=model.layers.13.self_attn.k_norm.weight] Loading weights: 49%|████▊ | 151/311 [00:00<00:00, 313.51it/s, Materializing param=model.layers.13.self_attn.k_norm.weight] Loading weights: 48%|████▊ | 150/311 [00:00<00:00, 345.22it/s, Materializing param=model.layers.13.post_attention_layernorm.weight] Loading weights: 49%|████▉ | 152/311 [00:00<00:00, 313.51it/s, Materializing param=model.layers.13.self_attn.k_proj.weight] Loading weights: 49%|████▉ | 152/311 [00:00<00:00, 313.51it/s, Materializing param=model.layers.13.self_attn.k_proj.weight] Loading weights: 49%|████▊ | 151/311 [00:00<00:00, 345.22it/s, Materializing param=model.layers.13.self_attn.k_norm.weight] Loading weights: 34%|███▍ | 105/311 [00:00<00:01, 182.18it/s, Materializing param=model.layers.9.mlp.up_proj.weight] Loading weights: 49%|████▊ | 151/311 [00:00<00:00, 345.22it/s, Materializing param=model.layers.13.self_attn.k_norm.weight] Loading weights: 34%|███▍ | 105/311 [00:00<00:01, 182.18it/s, Materializing param=model.layers.9.mlp.up_proj.weight] Loading weights: 44%|████▍ | 138/311 [00:00<00:00, 290.52it/s, Materializing param=model.layers.12.mlp.up_proj.weight] Loading weights: 49%|████▉ | 152/311 [00:00<00:00, 345.22it/s, Materializing param=model.layers.13.self_attn.k_proj.weight] Loading weights: 49%|████▉ | 152/311 [00:00<00:00, 345.22it/s, Materializing param=model.layers.13.self_attn.k_proj.weight] Loading weights: 44%|████▍ | 138/311 [00:00<00:00, 290.52it/s, Materializing param=model.layers.12.mlp.up_proj.weight] Loading weights: 49%|████▉ | 153/311 [00:00<00:00, 345.22it/s, Materializing param=model.layers.13.self_attn.o_proj.weight] Loading weights: 49%|████▉ | 153/311 [00:00<00:00, 345.22it/s, Materializing param=model.layers.13.self_attn.o_proj.weight] Loading weights: 20%|█▉ | 61/311 [00:00<00:02, 121.83it/s, Materializing param=model.layers.5.mlp.up_proj.weight] Loading weights: 20%|█▉ | 61/311 [00:00<00:02, 121.83it/s, Materializing param=model.layers.5.mlp.up_proj.weight] Loading weights: 50%|████▉ | 154/311 [00:00<00:00, 345.22it/s, Materializing param=model.layers.13.self_attn.q_norm.weight] Loading weights: 50%|████▉ | 154/311 [00:00<00:00, 345.22it/s, Materializing param=model.layers.13.self_attn.q_norm.weight] Loading weights: 50%|████▉ | 155/311 [00:00<00:00, 345.22it/s, Materializing param=model.layers.13.self_attn.q_proj.weight] Loading weights: 50%|████▉ | 155/311 [00:00<00:00, 345.22it/s, Materializing param=model.layers.13.self_attn.q_proj.weight] Loading weights: 49%|████▉ | 153/311 [00:00<00:00, 313.51it/s, Materializing param=model.layers.13.self_attn.o_proj.weight] Loading weights: 49%|████▉ | 153/311 [00:00<00:00, 313.51it/s, Materializing param=model.layers.13.self_attn.o_proj.weight] Loading weights: 50%|████▉ | 154/311 [00:00<00:00, 313.51it/s, Materializing param=model.layers.13.self_attn.q_norm.weight] Loading weights: 50%|████▉ | 154/311 [00:00<00:00, 313.51it/s, Materializing param=model.layers.13.self_attn.q_norm.weight] Loading weights: 50%|████▉ | 155/311 [00:00<00:00, 313.51it/s, Materializing param=model.layers.13.self_attn.q_proj.weight] Loading weights: 50%|████▉ | 155/311 [00:00<00:00, 313.51it/s, Materializing param=model.layers.13.self_attn.q_proj.weight] Loading weights: 50%|█████ | 156/311 [00:00<00:00, 313.51it/s, Materializing param=model.layers.13.self_attn.v_proj.weight] Loading weights: 50%|█████ | 156/311 [00:00<00:00, 345.22it/s, Materializing param=model.layers.13.self_attn.v_proj.weight] Loading weights: 50%|█████ | 156/311 [00:00<00:00, 313.51it/s, Materializing param=model.layers.13.self_attn.v_proj.weight] Loading weights: 50%|█████ | 156/311 [00:00<00:00, 345.22it/s, Materializing param=model.layers.13.self_attn.v_proj.weight] Loading weights: 44%|████▍ | 137/311 [00:00<00:00, 315.95it/s, Materializing param=model.layers.12.mlp.down_proj.weight] Loading weights: 44%|████▍ | 137/311 [00:00<00:00, 315.95it/s, Materializing param=model.layers.12.mlp.gate_proj.weight] Loading weights: 44%|████▍ | 137/311 [00:00<00:00, 315.95it/s, Materializing param=model.layers.12.mlp.gate_proj.weight] Loading weights: 50%|█████ | 157/311 [00:00<00:00, 313.51it/s, Materializing param=model.layers.14.input_layernorm.weight] Loading weights: 50%|█████ | 157/311 [00:00<00:00, 313.51it/s, Materializing param=model.layers.14.input_layernorm.weight] Loading weights: 50%|█████ | 157/311 [00:00<00:00, 345.22it/s, Materializing param=model.layers.14.input_layernorm.weight] Loading weights: 50%|█████ | 157/311 [00:00<00:00, 345.22it/s, Materializing param=model.layers.14.input_layernorm.weight] Loading weights: 51%|█████ | 158/311 [00:00<00:00, 345.22it/s, Materializing param=model.layers.14.mlp.down_proj.weight] Loading weights: 51%|█████ | 158/311 [00:00<00:00, 345.22it/s, Materializing param=model.layers.14.mlp.down_proj.weight] Loading weights: 51%|█████ | 158/311 [00:00<00:00, 313.51it/s, Materializing param=model.layers.14.mlp.down_proj.weight] Loading weights: 51%|█████ | 158/311 [00:00<00:00, 313.51it/s, Materializing param=model.layers.14.mlp.down_proj.weight] Loading weights: 30%|██▉ | 93/311 [00:00<00:01, 200.31it/s, Materializing param=model.layers.8.mlp.gate_proj.weight] Loading weights: 30%|██▉ | 93/311 [00:00<00:01, 200.31it/s, Materializing param=model.layers.8.mlp.gate_proj.weight] Loading weights: 45%|████▍ | 139/311 [00:00<00:00, 290.52it/s, Materializing param=model.layers.12.post_attention_layernorm.weight] Loading weights: 45%|████▍ | 139/311 [00:00<00:00, 290.52it/s, Materializing param=model.layers.12.post_attention_layernorm.weight] Loading weights: 45%|████▌ | 140/311 [00:00<00:00, 290.52it/s, Materializing param=model.layers.12.self_attn.k_norm.weight] Loading weights: 45%|████▌ | 140/311 [00:00<00:00, 290.52it/s, Materializing param=model.layers.12.self_attn.k_norm.weight] Loading weights: 34%|███▍ | 106/311 [00:00<00:01, 182.18it/s, Materializing param=model.layers.9.post_attention_layernorm.weight] Loading weights: 45%|████▌ | 141/311 [00:00<00:00, 290.52it/s, Materializing param=model.layers.12.self_attn.k_proj.weight] Loading weights: 34%|███▍ | 106/311 [00:00<00:01, 182.18it/s, Materializing param=model.layers.9.post_attention_layernorm.weight] Loading weights: 45%|████▌ | 141/311 [00:00<00:00, 290.52it/s, Materializing param=model.layers.12.self_attn.k_proj.weight] Loading weights: 34%|███▍ | 107/311 [00:00<00:01, 182.18it/s, Materializing param=model.layers.9.self_attn.k_norm.weight] Loading weights: 34%|███▍ | 107/311 [00:00<00:01, 182.18it/s, Materializing param=model.layers.9.self_attn.k_norm.weight] Loading weights: 35%|███▍ | 108/311 [00:00<00:01, 182.18it/s, Materializing param=model.layers.9.self_attn.k_proj.weight] Loading weights: 35%|███▍ | 108/311 [00:00<00:01, 182.18it/s, Materializing param=model.layers.9.self_attn.k_proj.weight] Loading weights: 46%|████▌ | 142/311 [00:00<00:00, 290.52it/s, Materializing param=model.layers.12.self_attn.o_proj.weight] Loading weights: 46%|████▌ | 142/311 [00:00<00:00, 290.52it/s, Materializing param=model.layers.12.self_attn.o_proj.weight] Loading weights: 20%|█▉ | 62/311 [00:00<00:02, 121.83it/s, Materializing param=model.layers.5.post_attention_layernorm.weight] Loading weights: 20%|█▉ | 62/311 [00:00<00:02, 121.83it/s, Materializing param=model.layers.5.post_attention_layernorm.weight] Loading weights: 35%|███▌ | 109/311 [00:00<00:01, 182.18it/s, Materializing param=model.layers.9.self_attn.o_proj.weight] Loading weights: 35%|███▌ | 109/311 [00:00<00:01, 182.18it/s, Materializing param=model.layers.9.self_attn.o_proj.weight] Loading weights: 20%|██ | 63/311 [00:00<00:02, 121.83it/s, Materializing param=model.layers.5.self_attn.k_norm.weight] Loading weights: 20%|██ | 63/311 [00:00<00:02, 121.83it/s, Materializing param=model.layers.5.self_attn.k_norm.weight] Loading weights: 21%|██ | 64/311 [00:00<00:02, 121.83it/s, Materializing param=model.layers.5.self_attn.k_proj.weight] Loading weights: 21%|██ | 64/311 [00:00<00:02, 121.83it/s, Materializing param=model.layers.5.self_attn.k_proj.weight] Loading weights: 21%|██ | 65/311 [00:00<00:02, 121.83it/s, Materializing param=model.layers.5.self_attn.o_proj.weight] Loading weights: 21%|██ | 65/311 [00:00<00:02, 121.83it/s, Materializing param=model.layers.5.self_attn.o_proj.weight] Loading weights: 44%|████▍ | 138/311 [00:00<00:00, 315.95it/s, Materializing param=model.layers.12.mlp.up_proj.weight] Loading weights: 46%|████▌ | 143/311 [00:00<00:00, 290.52it/s, Materializing param=model.layers.12.self_attn.q_norm.weight] Loading weights: 44%|████▍ | 138/311 [00:00<00:00, 315.95it/s, Materializing param=model.layers.12.mlp.up_proj.weight] Loading weights: 46%|████▌ | 143/311 [00:00<00:00, 290.52it/s, Materializing param=model.layers.12.self_attn.q_norm.weight] Loading weights: 46%|████▋ | 144/311 [00:00<00:00, 290.52it/s, Materializing param=model.layers.12.self_attn.q_proj.weight] Loading weights: 46%|████▋ | 144/311 [00:00<00:00, 290.52it/s, Materializing param=model.layers.12.self_attn.q_proj.weight] Loading weights: 35%|███▌ | 110/311 [00:00<00:00, 240.68it/s, Materializing param=model.layers.9.self_attn.o_proj.weight] Loading weights: 35%|███▌ | 110/311 [00:00<00:00, 240.68it/s, Materializing param=model.layers.9.self_attn.q_norm.weight] Loading weights: 35%|███▌ | 110/311 [00:00<00:00, 240.68it/s, Materializing param=model.layers.9.self_attn.q_norm.weight] Loading weights: 21%|██ | 66/311 [00:00<00:02, 121.83it/s, Materializing param=model.layers.5.self_attn.q_norm.weight] Loading weights: 21%|██ | 66/311 [00:00<00:02, 121.83it/s, Materializing param=model.layers.5.self_attn.q_norm.weight] Loading weights: 47%|████▋ | 145/311 [00:00<00:00, 290.52it/s, Materializing param=model.layers.12.self_attn.v_proj.weight] Loading weights: 51%|█████ | 159/311 [00:00<00:00, 313.51it/s, Materializing param=model.layers.14.mlp.gate_proj.weight] Loading weights: 47%|████▋ | 145/311 [00:00<00:00, 290.52it/s, Materializing param=model.layers.12.self_attn.v_proj.weight] Loading weights: 51%|█████ | 159/311 [00:00<00:00, 313.51it/s, Materializing param=model.layers.14.mlp.gate_proj.weight] Loading weights: 51%|█████ | 159/311 [00:00<00:00, 345.22it/s, Materializing param=model.layers.14.mlp.gate_proj.weight] Loading weights: 51%|█████ | 159/311 [00:00<00:00, 345.22it/s, Materializing param=model.layers.14.mlp.gate_proj.weight] Loading weights: 47%|████▋ | 146/311 [00:00<00:00, 290.52it/s, Materializing param=model.layers.13.input_layernorm.weight] Loading weights: 47%|████▋ | 146/311 [00:00<00:00, 290.52it/s, Materializing param=model.layers.13.input_layernorm.weight] Loading weights: 36%|███▌ | 111/311 [00:00<00:00, 240.68it/s, Materializing param=model.layers.9.self_attn.q_proj.weight] Loading weights: 36%|███▌ | 111/311 [00:00<00:00, 240.68it/s, Materializing param=model.layers.9.self_attn.q_proj.weight] Loading weights: 36%|███▌ | 112/311 [00:00<00:00, 240.68it/s, Materializing param=model.layers.9.self_attn.v_proj.weight] Loading weights: 30%|███ | 94/311 [00:00<00:01, 200.31it/s, Materializing param=model.layers.8.mlp.up_proj.weight] Loading weights: 36%|███▌ | 112/311 [00:00<00:00, 240.68it/s, Materializing param=model.layers.9.self_attn.v_proj.weight] Loading weights: 30%|███ | 94/311 [00:00<00:01, 200.31it/s, Materializing param=model.layers.8.mlp.up_proj.weight] Loading weights: 22%|██▏ | 67/311 [00:00<00:02, 121.83it/s, Materializing param=model.layers.5.self_attn.q_proj.weight] Loading weights: 22%|██▏ | 67/311 [00:00<00:02, 121.83it/s, Materializing param=model.layers.5.self_attn.q_proj.weight] Loading weights: 22%|██▏ | 68/311 [00:00<00:01, 121.83it/s, Materializing param=model.layers.5.self_attn.v_proj.weight] Loading weights: 36%|███▋ | 113/311 [00:00<00:00, 240.68it/s, Materializing param=model.layers.10.input_layernorm.weight] Loading weights: 22%|██▏ | 68/311 [00:00<00:01, 121.83it/s, Materializing param=model.layers.5.self_attn.v_proj.weight] Loading weights: 36%|███▋ | 113/311 [00:00<00:00, 240.68it/s, Materializing param=model.layers.10.input_layernorm.weight] Loading weights: 37%|███▋ | 114/311 [00:00<00:00, 240.68it/s, Materializing param=model.layers.10.mlp.down_proj.weight] Loading weights: 37%|███▋ | 114/311 [00:00<00:00, 240.68it/s, Materializing param=model.layers.10.mlp.down_proj.weight] Loading weights: 22%|██▏ | 69/311 [00:00<00:01, 121.83it/s, Materializing param=model.layers.6.input_layernorm.weight] Loading weights: 22%|██▏ | 69/311 [00:00<00:01, 121.83it/s, Materializing param=model.layers.6.input_layernorm.weight] Loading weights: 23%|██▎ | 70/311 [00:00<00:01, 121.83it/s, Materializing param=model.layers.6.mlp.down_proj.weight] Loading weights: 23%|██▎ | 70/311 [00:00<00:01, 121.83it/s, Materializing param=model.layers.6.mlp.down_proj.weight] Loading weights: 45%|████▍ | 139/311 [00:00<00:00, 315.95it/s, Materializing param=model.layers.12.post_attention_layernorm.weight] Loading weights: 45%|████▍ | 139/311 [00:00<00:00, 315.95it/s, Materializing param=model.layers.12.post_attention_layernorm.weight] Loading weights: 45%|████▌ | 140/311 [00:00<00:00, 315.95it/s, Materializing param=model.layers.12.self_attn.k_norm.weight] Loading weights: 45%|████▌ | 140/311 [00:00<00:00, 315.95it/s, Materializing param=model.layers.12.self_attn.k_norm.weight] Loading weights: 45%|████▌ | 141/311 [00:00<00:00, 315.95it/s, Materializing param=model.layers.12.self_attn.k_proj.weight] Loading weights: 45%|████▌ | 141/311 [00:00<00:00, 315.95it/s, Materializing param=model.layers.12.self_attn.k_proj.weight] Loading weights: 51%|█████▏ | 160/311 [00:00<00:00, 313.51it/s, Materializing param=model.layers.14.mlp.up_proj.weight] Loading weights: 46%|████▌ | 142/311 [00:00<00:00, 315.95it/s, Materializing param=model.layers.12.self_attn.o_proj.weight] Loading weights: 51%|█████▏ | 160/311 [00:00<00:00, 313.51it/s, Materializing param=model.layers.14.mlp.up_proj.weight] Loading weights: 51%|█████▏ | 160/311 [00:00<00:00, 345.22it/s, Materializing param=model.layers.14.mlp.up_proj.weight] Loading weights: 46%|████▌ | 142/311 [00:00<00:00, 315.95it/s, Materializing param=model.layers.12.self_attn.o_proj.weight] Loading weights: 51%|█████▏ | 160/311 [00:00<00:00, 345.22it/s, Materializing param=model.layers.14.mlp.up_proj.weight] Loading weights: 46%|████▌ | 143/311 [00:00<00:00, 315.95it/s, Materializing param=model.layers.12.self_attn.q_norm.weight] Loading weights: 46%|████▌ | 143/311 [00:00<00:00, 315.95it/s, Materializing param=model.layers.12.self_attn.q_norm.weight] Loading weights: 46%|████▋ | 144/311 [00:00<00:00, 315.95it/s, Materializing param=model.layers.12.self_attn.q_proj.weight] Loading weights: 46%|████▋ | 144/311 [00:00<00:00, 315.95it/s, Materializing param=model.layers.12.self_attn.q_proj.weight] Loading weights: 47%|████▋ | 147/311 [00:00<00:00, 290.52it/s, Materializing param=model.layers.13.mlp.down_proj.weight] Loading weights: 47%|████▋ | 147/311 [00:00<00:00, 290.52it/s, Materializing param=model.layers.13.mlp.down_proj.weight] Loading weights: 48%|████▊ | 148/311 [00:00<00:00, 290.52it/s, Materializing param=model.layers.13.mlp.gate_proj.weight] Loading weights: 48%|████▊ | 148/311 [00:00<00:00, 290.52it/s, Materializing param=model.layers.13.mlp.gate_proj.weight] Loading weights: 47%|████▋ | 145/311 [00:00<00:00, 315.95it/s, Materializing param=model.layers.12.self_attn.v_proj.weight] Loading weights: 47%|████▋ | 145/311 [00:00<00:00, 315.95it/s, Materializing param=model.layers.12.self_attn.v_proj.weight] Loading weights: 31%|███ | 95/311 [00:00<00:01, 200.31it/s, Materializing param=model.layers.8.post_attention_layernorm.weight] Loading weights: 31%|███ | 95/311 [00:00<00:01, 200.31it/s, Materializing param=model.layers.8.post_attention_layernorm.weight] Loading weights: 31%|███ | 96/311 [00:00<00:01, 200.31it/s, Materializing param=model.layers.8.self_attn.k_norm.weight] Loading weights: 31%|███ | 96/311 [00:00<00:01, 200.31it/s, Materializing param=model.layers.8.self_attn.k_norm.weight] Loading weights: 23%|██▎ | 71/311 [00:00<00:01, 121.83it/s, Materializing param=model.layers.6.mlp.gate_proj.weight] Loading weights: 23%|██▎ | 71/311 [00:00<00:01, 121.83it/s, Materializing param=model.layers.6.mlp.gate_proj.weight] Loading weights: 31%|███ | 97/311 [00:00<00:01, 200.31it/s, Materializing param=model.layers.8.self_attn.k_proj.weight] Loading weights: 31%|███ | 97/311 [00:00<00:01, 200.31it/s, Materializing param=model.layers.8.self_attn.k_proj.weight] Loading weights: 47%|████▋ | 146/311 [00:00<00:00, 315.95it/s, Materializing param=model.layers.13.input_layernorm.weight] Loading weights: 47%|████▋ | 146/311 [00:00<00:00, 315.95it/s, Materializing param=model.layers.13.input_layernorm.weight] Loading weights: 47%|████▋ | 147/311 [00:00<00:00, 315.95it/s, Materializing param=model.layers.13.mlp.down_proj.weight] Loading weights: 47%|████▋ | 147/311 [00:00<00:00, 315.95it/s, Materializing param=model.layers.13.mlp.down_proj.weight] Loading weights: 32%|███▏ | 98/311 [00:00<00:01, 200.31it/s, Materializing param=model.layers.8.self_attn.o_proj.weight] Loading weights: 32%|███▏ | 98/311 [00:00<00:01, 200.31it/s, Materializing param=model.layers.8.self_attn.o_proj.weight] Loading weights: 32%|███▏ | 99/311 [00:00<00:01, 200.31it/s, Materializing param=model.layers.8.self_attn.q_norm.weight] Loading weights: 32%|███▏ | 99/311 [00:00<00:01, 200.31it/s, Materializing param=model.layers.8.self_attn.q_norm.weight] Loading weights: 32%|███▏ | 100/311 [00:00<00:01, 200.31it/s, Materializing param=model.layers.8.self_attn.q_proj.weight] Loading weights: 32%|███▏ | 100/311 [00:00<00:01, 200.31it/s, Materializing param=model.layers.8.self_attn.q_proj.weight] Loading weights: 48%|████▊ | 149/311 [00:00<00:00, 290.52it/s, Materializing param=model.layers.13.mlp.up_proj.weight] Loading weights: 48%|████▊ | 149/311 [00:00<00:00, 290.52it/s, Materializing param=model.layers.13.mlp.up_proj.weight] Loading weights: 52%|█████▏ | 161/311 [00:00<00:00, 345.22it/s, Materializing param=model.layers.14.post_attention_layernorm.weight] Loading weights: 52%|█████▏ | 161/311 [00:00<00:00, 345.22it/s, Materializing param=model.layers.14.post_attention_layernorm.weight] Loading weights: 52%|█████▏ | 161/311 [00:00<00:00, 313.51it/s, Materializing param=model.layers.14.post_attention_layernorm.weight] Loading weights: 52%|█████▏ | 161/311 [00:00<00:00, 313.51it/s, Materializing param=model.layers.14.post_attention_layernorm.weight] Loading weights: 52%|█████▏ | 162/311 [00:00<00:00, 345.22it/s, Materializing param=model.layers.14.self_attn.k_norm.weight] Loading weights: 52%|█████▏ | 162/311 [00:00<00:00, 345.22it/s, Materializing param=model.layers.14.self_attn.k_norm.weight] Loading weights: 52%|█████▏ | 162/311 [00:00<00:00, 313.51it/s, Materializing param=model.layers.14.self_attn.k_norm.weight] Loading weights: 52%|█████▏ | 162/311 [00:00<00:00, 313.51it/s, Materializing param=model.layers.14.self_attn.k_norm.weight] Loading weights: 52%|█████▏ | 163/311 [00:00<00:00, 345.22it/s, Materializing param=model.layers.14.self_attn.k_proj.weight] Loading weights: 32%|███▏ | 101/311 [00:00<00:01, 200.31it/s, Materializing param=model.layers.8.self_attn.v_proj.weight] Loading weights: 52%|█████▏ | 163/311 [00:00<00:00, 345.22it/s, Materializing param=model.layers.14.self_attn.k_proj.weight] Loading weights: 52%|█████▏ | 163/311 [00:00<00:00, 313.51it/s, Materializing param=model.layers.14.self_attn.k_proj.weight] Loading weights: 32%|███▏ | 101/311 [00:00<00:01, 200.31it/s, Materializing param=model.layers.8.self_attn.v_proj.weight] Loading weights: 52%|█████▏ | 163/311 [00:00<00:00, 313.51it/s, Materializing param=model.layers.14.self_attn.k_proj.weight] Loading weights: 48%|████▊ | 148/311 [00:00<00:00, 315.95it/s, Materializing param=model.layers.13.mlp.gate_proj.weight] Loading weights: 48%|████▊ | 148/311 [00:00<00:00, 315.95it/s, Materializing param=model.layers.13.mlp.gate_proj.weight] Loading weights: 53%|█████▎ | 164/311 [00:00<00:00, 313.51it/s, Materializing param=model.layers.14.self_attn.o_proj.weight] Loading weights: 53%|█████▎ | 164/311 [00:00<00:00, 345.22it/s, Materializing param=model.layers.14.self_attn.o_proj.weight] Loading weights: 53%|█████▎ | 164/311 [00:00<00:00, 313.51it/s, Materializing param=model.layers.14.self_attn.o_proj.weight] Loading weights: 53%|█████▎ | 164/311 [00:00<00:00, 345.22it/s, Materializing param=model.layers.14.self_attn.o_proj.weight] Loading weights: 33%|███▎ | 102/311 [00:00<00:01, 200.31it/s, Materializing param=model.layers.9.input_layernorm.weight] Loading weights: 33%|███▎ | 102/311 [00:00<00:01, 200.31it/s, Materializing param=model.layers.9.input_layernorm.weight] Loading weights: 33%|███▎ | 103/311 [00:00<00:01, 200.31it/s, Materializing param=model.layers.9.mlp.down_proj.weight] Loading weights: 33%|███▎ | 103/311 [00:00<00:01, 200.31it/s, Materializing param=model.layers.9.mlp.down_proj.weight] Loading weights: 23%|██▎ | 72/311 [00:00<00:01, 121.83it/s, Materializing param=model.layers.6.mlp.up_proj.weight] Loading weights: 37%|███▋ | 115/311 [00:00<00:00, 240.68it/s, Materializing param=model.layers.10.mlp.gate_proj.weight] Loading weights: 23%|██▎ | 72/311 [00:00<00:01, 121.83it/s, Materializing param=model.layers.6.mlp.up_proj.weight] Loading weights: 37%|███▋ | 115/311 [00:00<00:00, 240.68it/s, Materializing param=model.layers.10.mlp.gate_proj.weight] Loading weights: 37%|███▋ | 116/311 [00:00<00:00, 240.68it/s, Materializing param=model.layers.10.mlp.up_proj.weight] Loading weights: 37%|███▋ | 116/311 [00:00<00:00, 240.68it/s, Materializing param=model.layers.10.mlp.up_proj.weight] Loading weights: 53%|█████▎ | 165/311 [00:00<00:00, 345.22it/s, Materializing param=model.layers.14.self_attn.q_norm.weight] Loading weights: 53%|█████▎ | 165/311 [00:00<00:00, 345.22it/s, Materializing param=model.layers.14.self_attn.q_norm.weight] Loading weights: 53%|█████▎ | 165/311 [00:00<00:00, 313.51it/s, Materializing param=model.layers.14.self_attn.q_norm.weight] Loading weights: 53%|█████▎ | 165/311 [00:00<00:00, 313.51it/s, Materializing param=model.layers.14.self_attn.q_norm.weight] Loading weights: 53%|█████▎ | 166/311 [00:00<00:00, 313.51it/s, Materializing param=model.layers.14.self_attn.q_proj.weight] Loading weights: 53%|█████▎ | 166/311 [00:00<00:00, 345.22it/s, Materializing param=model.layers.14.self_attn.q_proj.weight] Loading weights: 53%|█████▎ | 166/311 [00:00<00:00, 313.51it/s, Materializing param=model.layers.14.self_attn.q_proj.weight] Loading weights: 53%|█████▎ | 166/311 [00:00<00:00, 345.22it/s, Materializing param=model.layers.14.self_attn.q_proj.weight] Loading weights: 54%|█████▎ | 167/311 [00:00<00:00, 313.51it/s, Materializing param=model.layers.14.self_attn.v_proj.weight] Loading weights: 54%|█████▎ | 167/311 [00:00<00:00, 345.22it/s, Materializing param=model.layers.14.self_attn.v_proj.weight] Loading weights: 54%|█████▎ | 167/311 [00:00<00:00, 313.51it/s, Materializing param=model.layers.14.self_attn.v_proj.weight] Loading weights: 54%|█████▎ | 167/311 [00:00<00:00, 345.22it/s, Materializing param=model.layers.14.self_attn.v_proj.weight] Loading weights: 48%|████▊ | 150/311 [00:00<00:00, 290.52it/s, Materializing param=model.layers.13.post_attention_layernorm.weight] Loading weights: 48%|████▊ | 150/311 [00:00<00:00, 290.52it/s, Materializing param=model.layers.13.post_attention_layernorm.weight] Loading weights: 54%|█████▍ | 168/311 [00:00<00:00, 313.51it/s, Materializing param=model.layers.15.input_layernorm.weight] Loading weights: 49%|████▊ | 151/311 [00:00<00:00, 290.52it/s, Materializing param=model.layers.13.self_attn.k_norm.weight] Loading weights: 54%|█████▍ | 168/311 [00:00<00:00, 313.51it/s, Materializing param=model.layers.15.input_layernorm.weight] Loading weights: 54%|█████▍ | 168/311 [00:00<00:00, 345.22it/s, Materializing param=model.layers.15.input_layernorm.weight] Loading weights: 49%|████▊ | 151/311 [00:00<00:00, 290.52it/s, Materializing param=model.layers.13.self_attn.k_norm.weight] Loading weights: 54%|█████▍ | 168/311 [00:00<00:00, 345.22it/s, Materializing param=model.layers.15.input_layernorm.weight] Loading weights: 54%|█████▍ | 169/311 [00:00<00:00, 345.22it/s, Materializing param=model.layers.15.mlp.down_proj.weight] Loading weights: 54%|█████▍ | 169/311 [00:00<00:00, 345.22it/s, Materializing param=model.layers.15.mlp.down_proj.weight] Loading weights: 49%|████▉ | 152/311 [00:00<00:00, 290.52it/s, Materializing param=model.layers.13.self_attn.k_proj.weight] Loading weights: 49%|████▉ | 152/311 [00:00<00:00, 290.52it/s, Materializing param=model.layers.13.self_attn.k_proj.weight] Loading weights: 49%|████▉ | 153/311 [00:00<00:00, 290.52it/s, Materializing param=model.layers.13.self_attn.o_proj.weight] Loading weights: 48%|████▊ | 149/311 [00:00<00:00, 315.95it/s, Materializing param=model.layers.13.mlp.up_proj.weight] Loading weights: 48%|████▊ | 149/311 [00:00<00:00, 315.95it/s, Materializing param=model.layers.13.mlp.up_proj.weight] Loading weights: 49%|████▉ | 153/311 [00:00<00:00, 290.52it/s, Materializing param=model.layers.13.self_attn.o_proj.weight] Loading weights: 33%|███▎ | 104/311 [00:00<00:01, 200.31it/s, Materializing param=model.layers.9.mlp.gate_proj.weight] Loading weights: 33%|███▎ | 104/311 [00:00<00:01, 200.31it/s, Materializing param=model.layers.9.mlp.gate_proj.weight] Loading weights: 50%|████▉ | 154/311 [00:00<00:00, 290.52it/s, Materializing param=model.layers.13.self_attn.q_norm.weight] Loading weights: 38%|███▊ | 117/311 [00:00<00:00, 240.68it/s, Materializing param=model.layers.10.post_attention_layernorm.weight] Loading weights: 50%|████▉ | 154/311 [00:00<00:00, 290.52it/s, Materializing param=model.layers.13.self_attn.q_norm.weight] Loading weights: 38%|███▊ | 117/311 [00:00<00:00, 240.68it/s, Materializing param=model.layers.10.post_attention_layernorm.weight] Loading weights: 23%|██▎ | 73/311 [00:00<00:01, 121.83it/s, Materializing param=model.layers.6.post_attention_layernorm.weight] Loading weights: 23%|██▎ | 73/311 [00:00<00:01, 121.83it/s, Materializing param=model.layers.6.post_attention_layernorm.weight] Loading weights: 38%|███▊ | 118/311 [00:00<00:00, 240.68it/s, Materializing param=model.layers.10.self_attn.k_norm.weight] Loading weights: 50%|████▉ | 155/311 [00:00<00:00, 290.52it/s, Materializing param=model.layers.13.self_attn.q_proj.weight] Loading weights: 38%|███▊ | 118/311 [00:00<00:00, 240.68it/s, Materializing param=model.layers.10.self_attn.k_norm.weight] Loading weights: 24%|██▍ | 74/311 [00:00<00:01, 121.83it/s, Materializing param=model.layers.6.self_attn.k_norm.weight] Loading weights: 50%|████▉ | 155/311 [00:00<00:00, 290.52it/s, Materializing param=model.layers.13.self_attn.q_proj.weight] Loading weights: 24%|██▍ | 74/311 [00:00<00:01, 121.83it/s, Materializing param=model.layers.6.self_attn.k_norm.weight] Loading weights: 24%|██▍ | 75/311 [00:00<00:01, 121.83it/s, Materializing param=model.layers.6.self_attn.k_proj.weight] Loading weights: 24%|██▍ | 75/311 [00:00<00:01, 121.83it/s, Materializing param=model.layers.6.self_attn.k_proj.weight] Loading weights: 38%|███▊ | 119/311 [00:00<00:00, 240.68it/s, Materializing param=model.layers.10.self_attn.k_proj.weight] Loading weights: 38%|███▊ | 119/311 [00:00<00:00, 240.68it/s, Materializing param=model.layers.10.self_attn.k_proj.weight] Loading weights: 24%|██▍ | 76/311 [00:00<00:01, 121.83it/s, Materializing param=model.layers.6.self_attn.o_proj.weight] Loading weights: 24%|██▍ | 76/311 [00:00<00:01, 121.83it/s, Materializing param=model.layers.6.self_attn.o_proj.weight] Loading weights: 39%|███▊ | 120/311 [00:00<00:00, 240.68it/s, Materializing param=model.layers.10.self_attn.o_proj.weight] Loading weights: 39%|███▊ | 120/311 [00:00<00:00, 240.68it/s, Materializing param=model.layers.10.self_attn.o_proj.weight] Loading weights: 50%|█████ | 156/311 [00:00<00:00, 290.52it/s, Materializing param=model.layers.13.self_attn.v_proj.weight] Loading weights: 50%|█████ | 156/311 [00:00<00:00, 290.52it/s, Materializing param=model.layers.13.self_attn.v_proj.weight] Loading weights: 25%|██▍ | 77/311 [00:00<00:01, 121.83it/s, Materializing param=model.layers.6.self_attn.q_norm.weight] Loading weights: 50%|█████ | 157/311 [00:00<00:00, 290.52it/s, Materializing param=model.layers.14.input_layernorm.weight] Loading weights: 25%|██▍ | 77/311 [00:00<00:01, 121.83it/s, Materializing param=model.layers.6.self_attn.q_norm.weight] Loading weights: 50%|█████ | 157/311 [00:00<00:00, 290.52it/s, Materializing param=model.layers.14.input_layernorm.weight] Loading weights: 25%|██▌ | 78/311 [00:00<00:01, 121.83it/s, Materializing param=model.layers.6.self_attn.q_proj.weight] Loading weights: 25%|██▌ | 78/311 [00:00<00:01, 121.83it/s, Materializing param=model.layers.6.self_attn.q_proj.weight] Loading weights: 51%|█████ | 158/311 [00:00<00:00, 290.52it/s, Materializing param=model.layers.14.mlp.down_proj.weight] Loading weights: 51%|█████ | 158/311 [00:00<00:00, 290.52it/s, Materializing param=model.layers.14.mlp.down_proj.weight] Loading weights: 48%|████▊ | 150/311 [00:00<00:00, 315.95it/s, Materializing param=model.layers.13.post_attention_layernorm.weight] Loading weights: 48%|████▊ | 150/311 [00:00<00:00, 315.95it/s, Materializing param=model.layers.13.post_attention_layernorm.weight] Loading weights: 39%|███▉ | 121/311 [00:00<00:00, 240.68it/s, Materializing param=model.layers.10.self_attn.q_norm.weight] Loading weights: 39%|███▉ | 121/311 [00:00<00:00, 240.68it/s, Materializing param=model.layers.10.self_attn.q_norm.weight] Loading weights: 55%|█████▍ | 170/311 [00:00<00:00, 345.22it/s, Materializing param=model.layers.15.mlp.gate_proj.weight] Loading weights: 49%|████▊ | 151/311 [00:00<00:00, 315.95it/s, Materializing param=model.layers.13.self_attn.k_norm.weight] Loading weights: 55%|█████▍ | 170/311 [00:00<00:00, 345.22it/s, Materializing param=model.layers.15.mlp.gate_proj.weight] Loading weights: 49%|████▊ | 151/311 [00:00<00:00, 315.95it/s, Materializing param=model.layers.13.self_attn.k_norm.weight] Loading weights: 54%|█████▍ | 169/311 [00:00<00:00, 313.51it/s, Materializing param=model.layers.15.mlp.down_proj.weight] Loading weights: 54%|█████▍ | 169/311 [00:00<00:00, 313.51it/s, Materializing param=model.layers.15.mlp.down_proj.weight] Loading weights: 39%|███▉ | 122/311 [00:00<00:00, 240.68it/s, Materializing param=model.layers.10.self_attn.q_proj.weight] Loading weights: 49%|████▉ | 152/311 [00:00<00:00, 315.95it/s, Materializing param=model.layers.13.self_attn.k_proj.weight] Loading weights: 39%|███▉ | 122/311 [00:00<00:00, 240.68it/s, Materializing param=model.layers.10.self_attn.q_proj.weight] Loading weights: 49%|████▉ | 152/311 [00:00<00:00, 315.95it/s, Materializing param=model.layers.13.self_attn.k_proj.weight] Loading weights: 55%|█████▍ | 170/311 [00:00<00:00, 313.51it/s, Materializing param=model.layers.15.mlp.gate_proj.weight] Loading weights: 55%|█████▍ | 170/311 [00:00<00:00, 313.51it/s, Materializing param=model.layers.15.mlp.gate_proj.weight] Loading weights: 49%|████▉ | 153/311 [00:00<00:00, 315.95it/s, Materializing param=model.layers.13.self_attn.o_proj.weight] Loading weights: 49%|████▉ | 153/311 [00:00<00:00, 315.95it/s, Materializing param=model.layers.13.self_attn.o_proj.weight] Loading weights: 25%|██▌ | 79/311 [00:00<00:01, 121.83it/s, Materializing param=model.layers.6.self_attn.v_proj.weight] Loading weights: 25%|██▌ | 79/311 [00:00<00:01, 121.83it/s, Materializing param=model.layers.6.self_attn.v_proj.weight] Loading weights: 40%|███▉ | 123/311 [00:00<00:00, 240.68it/s, Materializing param=model.layers.10.self_attn.v_proj.weight] Loading weights: 40%|███▉ | 123/311 [00:00<00:00, 240.68it/s, Materializing param=model.layers.10.self_attn.v_proj.weight] Loading weights: 34%|███▍ | 105/311 [00:00<00:01, 200.31it/s, Materializing param=model.layers.9.mlp.up_proj.weight] Loading weights: 34%|███▍ | 105/311 [00:00<00:01, 200.31it/s, Materializing param=model.layers.9.mlp.up_proj.weight] Loading weights: 26%|██▌ | 80/311 [00:00<00:01, 121.83it/s, Materializing param=model.layers.7.input_layernorm.weight] Loading weights: 26%|██▌ | 80/311 [00:00<00:01, 121.83it/s, Materializing param=model.layers.7.input_layernorm.weight] Loading weights: 26%|██▌ | 81/311 [00:00<00:01, 121.83it/s, Materializing param=model.layers.7.mlp.down_proj.weight] Loading weights: 26%|██▌ | 81/311 [00:00<00:01, 121.83it/s, Materializing param=model.layers.7.mlp.down_proj.weight] Loading weights: 50%|████▉ | 154/311 [00:00<00:00, 315.95it/s, Materializing param=model.layers.13.self_attn.q_norm.weight] Loading weights: 50%|████▉ | 154/311 [00:00<00:00, 315.95it/s, Materializing param=model.layers.13.self_attn.q_norm.weight] Loading weights: 40%|███▉ | 124/311 [00:00<00:00, 240.68it/s, Materializing param=model.layers.11.input_layernorm.weight] Loading weights: 40%|███▉ | 124/311 [00:00<00:00, 240.68it/s, Materializing param=model.layers.11.input_layernorm.weight] Loading weights: 40%|████ | 125/311 [00:00<00:00, 240.68it/s, Materializing param=model.layers.11.mlp.down_proj.weight] Loading weights: 40%|████ | 125/311 [00:00<00:00, 240.68it/s, Materializing param=model.layers.11.mlp.down_proj.weight] Loading weights: 50%|████▉ | 155/311 [00:00<00:00, 315.95it/s, Materializing param=model.layers.13.self_attn.q_proj.weight] Loading weights: 50%|████▉ | 155/311 [00:00<00:00, 315.95it/s, Materializing param=model.layers.13.self_attn.q_proj.weight] Loading weights: 50%|█████ | 156/311 [00:00<00:00, 315.95it/s, Materializing param=model.layers.13.self_attn.v_proj.weight] Loading weights: 50%|█████ | 156/311 [00:00<00:00, 315.95it/s, Materializing param=model.layers.13.self_attn.v_proj.weight] Loading weights: 51%|█████ | 159/311 [00:00<00:00, 290.52it/s, Materializing param=model.layers.14.mlp.gate_proj.weight] Loading weights: 51%|█████ | 159/311 [00:00<00:00, 290.52it/s, Materializing param=model.layers.14.mlp.gate_proj.weight] Loading weights: 50%|█████ | 157/311 [00:00<00:00, 315.95it/s, Materializing param=model.layers.14.input_layernorm.weight] Loading weights: 50%|█████ | 157/311 [00:00<00:00, 315.95it/s, Materializing param=model.layers.14.input_layernorm.weight] Loading weights: 51%|█████ | 158/311 [00:00<00:00, 315.95it/s, Materializing param=model.layers.14.mlp.down_proj.weight] Loading weights: 51%|█████ | 158/311 [00:00<00:00, 315.95it/s, Materializing param=model.layers.14.mlp.down_proj.weight] Loading weights: 55%|█████▍ | 171/311 [00:00<00:00, 313.51it/s, Materializing param=model.layers.15.mlp.up_proj.weight] Loading weights: 55%|█████▍ | 171/311 [00:00<00:00, 313.51it/s, Materializing param=model.layers.15.mlp.up_proj.weight] Loading weights: 55%|█████▍ | 171/311 [00:00<00:00, 345.22it/s, Materializing param=model.layers.15.mlp.up_proj.weight] Loading weights: 55%|█████▍ | 171/311 [00:00<00:00, 345.22it/s, Materializing param=model.layers.15.mlp.up_proj.weight] Loading weights: 26%|██▋ | 82/311 [00:00<00:01, 216.05it/s, Materializing param=model.layers.7.mlp.down_proj.weight] Loading weights: 26%|██▋ | 82/311 [00:00<00:01, 216.05it/s, Materializing param=model.layers.7.mlp.gate_proj.weight] Loading weights: 26%|██▋ | 82/311 [00:00<00:01, 216.05it/s, Materializing param=model.layers.7.mlp.gate_proj.weight] Loading weights: 34%|███▍ | 106/311 [00:00<00:01, 200.31it/s, Materializing param=model.layers.9.post_attention_layernorm.weight] Loading weights: 34%|███▍ | 106/311 [00:00<00:01, 200.31it/s, Materializing param=model.layers.9.post_attention_layernorm.weight] Loading weights: 34%|███▍ | 107/311 [00:00<00:01, 200.31it/s, Materializing param=model.layers.9.self_attn.k_norm.weight] Loading weights: 41%|████ | 126/311 [00:00<00:00, 240.68it/s, Materializing param=model.layers.11.mlp.gate_proj.weight] Loading weights: 34%|███▍ | 107/311 [00:00<00:01, 200.31it/s, Materializing param=model.layers.9.self_attn.k_norm.weight] Loading weights: 41%|████ | 126/311 [00:00<00:00, 240.68it/s, Materializing param=model.layers.11.mlp.gate_proj.weight] Loading weights: 35%|███▍ | 108/311 [00:00<00:01, 200.31it/s, Materializing param=model.layers.9.self_attn.k_proj.weight] Loading weights: 35%|███▍ | 108/311 [00:00<00:01, 200.31it/s, Materializing param=model.layers.9.self_attn.k_proj.weight] Loading weights: 35%|███▌ | 109/311 [00:00<00:01, 200.31it/s, Materializing param=model.layers.9.self_attn.o_proj.weight] Loading weights: 35%|███▌ | 109/311 [00:00<00:01, 200.31it/s, Materializing param=model.layers.9.self_attn.o_proj.weight] Loading weights: 51%|█████ | 159/311 [00:00<00:00, 315.95it/s, Materializing param=model.layers.14.mlp.gate_proj.weight] Loading weights: 51%|█████ | 159/311 [00:00<00:00, 315.95it/s, Materializing param=model.layers.14.mlp.gate_proj.weight] Loading weights: 51%|█████▏ | 160/311 [00:00<00:00, 290.52it/s, Materializing param=model.layers.14.mlp.up_proj.weight] Loading weights: 51%|█████▏ | 160/311 [00:00<00:00, 290.52it/s, Materializing param=model.layers.14.mlp.up_proj.weight] Loading weights: 35%|███▌ | 110/311 [00:00<00:01, 200.31it/s, Materializing param=model.layers.9.self_attn.q_norm.weight] Loading weights: 35%|███▌ | 110/311 [00:00<00:01, 200.31it/s, Materializing param=model.layers.9.self_attn.q_norm.weight] Loading weights: 36%|███▌ | 111/311 [00:00<00:00, 200.31it/s, Materializing param=model.layers.9.self_attn.q_proj.weight] Loading weights: 36%|███▌ | 111/311 [00:00<00:00, 200.31it/s, Materializing param=model.layers.9.self_attn.q_proj.weight] Loading weights: 55%|█████▌ | 172/311 [00:00<00:00, 313.51it/s, Materializing param=model.layers.15.post_attention_layernorm.weight] Loading weights: 55%|█████▌ | 172/311 [00:00<00:00, 345.22it/s, Materializing param=model.layers.15.post_attention_layernorm.weight] Loading weights: 55%|█████▌ | 172/311 [00:00<00:00, 313.51it/s, Materializing param=model.layers.15.post_attention_layernorm.weight] Loading weights: 55%|█████▌ | 172/311 [00:00<00:00, 345.22it/s, Materializing param=model.layers.15.post_attention_layernorm.weight] Loading weights: 56%|█████▌ | 173/311 [00:00<00:00, 313.51it/s, Materializing param=model.layers.15.self_attn.k_norm.weight] Loading weights: 56%|█████▌ | 173/311 [00:00<00:00, 345.22it/s, Materializing param=model.layers.15.self_attn.k_norm.weight] Loading weights: 56%|█████▌ | 173/311 [00:00<00:00, 313.51it/s, Materializing param=model.layers.15.self_attn.k_norm.weight] Loading weights: 56%|█████▌ | 173/311 [00:00<00:00, 345.22it/s, Materializing param=model.layers.15.self_attn.k_norm.weight] Loading weights: 56%|█████▌ | 174/311 [00:00<00:00, 313.51it/s, Materializing param=model.layers.15.self_attn.k_proj.weight] Loading weights: 56%|█████▌ | 174/311 [00:00<00:00, 345.22it/s, Materializing param=model.layers.15.self_attn.k_proj.weight] Loading weights: 56%|█████▌ | 174/311 [00:00<00:00, 313.51it/s, Materializing param=model.layers.15.self_attn.k_proj.weight] Loading weights: 56%|█████▌ | 174/311 [00:00<00:00, 345.22it/s, Materializing param=model.layers.15.self_attn.k_proj.weight] Loading weights: 56%|█████▋ | 175/311 [00:00<00:00, 313.51it/s, Materializing param=model.layers.15.self_attn.o_proj.weight] Loading weights: 56%|█████▋ | 175/311 [00:00<00:00, 345.22it/s, Materializing param=model.layers.15.self_attn.o_proj.weight] Loading weights: 56%|█████▋ | 175/311 [00:00<00:00, 313.51it/s, Materializing param=model.layers.15.self_attn.o_proj.weight] Loading weights: 56%|█████▋ | 175/311 [00:00<00:00, 345.22it/s, Materializing param=model.layers.15.self_attn.o_proj.weight] Loading weights: 36%|███▌ | 112/311 [00:00<00:00, 200.31it/s, Materializing param=model.layers.9.self_attn.v_proj.weight] Loading weights: 36%|███▌ | 112/311 [00:00<00:00, 200.31it/s, Materializing param=model.layers.9.self_attn.v_proj.weight] Loading weights: 27%|██▋ | 83/311 [00:00<00:01, 216.05it/s, Materializing param=model.layers.7.mlp.up_proj.weight] Loading weights: 27%|██▋ | 83/311 [00:00<00:01, 216.05it/s, Materializing param=model.layers.7.mlp.up_proj.weight] Loading weights: 36%|███▋ | 113/311 [00:00<00:00, 200.31it/s, Materializing param=model.layers.10.input_layernorm.weight] Loading weights: 36%|███▋ | 113/311 [00:00<00:00, 200.31it/s, Materializing param=model.layers.10.input_layernorm.weight] Loading weights: 37%|███▋ | 114/311 [00:00<00:00, 200.31it/s, Materializing param=model.layers.10.mlp.down_proj.weight] Loading weights: 57%|█████▋ | 176/311 [00:00<00:00, 345.22it/s, Materializing param=model.layers.15.self_attn.q_norm.weight] Loading weights: 37%|███▋ | 114/311 [00:00<00:00, 200.31it/s, Materializing param=model.layers.10.mlp.down_proj.weight] Loading weights: 57%|█████▋ | 176/311 [00:00<00:00, 345.22it/s, Materializing param=model.layers.15.self_attn.q_norm.weight] Loading weights: 57%|█████▋ | 176/311 [00:00<00:00, 313.51it/s, Materializing param=model.layers.15.self_attn.q_norm.weight] Loading weights: 57%|█████▋ | 176/311 [00:00<00:00, 313.51it/s, Materializing param=model.layers.15.self_attn.q_norm.weight] Loading weights: 57%|█████▋ | 177/311 [00:00<00:00, 345.22it/s, Materializing param=model.layers.15.self_attn.q_proj.weight] Loading weights: 57%|█████▋ | 177/311 [00:00<00:00, 345.22it/s, Materializing param=model.layers.15.self_attn.q_proj.weight] Loading weights: 57%|█████▋ | 177/311 [00:00<00:00, 313.51it/s, Materializing param=model.layers.15.self_attn.q_proj.weight] Loading weights: 57%|█████▋ | 177/311 [00:00<00:00, 313.51it/s, Materializing param=model.layers.15.self_attn.q_proj.weight] Loading weights: 41%|████ | 127/311 [00:00<00:00, 240.68it/s, Materializing param=model.layers.11.mlp.up_proj.weight] Loading weights: 51%|█████▏ | 160/311 [00:00<00:00, 315.95it/s, Materializing param=model.layers.14.mlp.up_proj.weight] Loading weights: 41%|████ | 127/311 [00:00<00:00, 240.68it/s, Materializing param=model.layers.11.mlp.up_proj.weight] Loading weights: 51%|█████▏ | 160/311 [00:00<00:00, 315.95it/s, Materializing param=model.layers.14.mlp.up_proj.weight] Loading weights: 52%|█████▏ | 161/311 [00:00<00:00, 290.52it/s, Materializing param=model.layers.14.post_attention_layernorm.weight] Loading weights: 52%|█████▏ | 161/311 [00:00<00:00, 290.52it/s, Materializing param=model.layers.14.post_attention_layernorm.weight] Loading weights: 57%|█████▋ | 178/311 [00:00<00:00, 313.51it/s, Materializing param=model.layers.15.self_attn.v_proj.weight] Loading weights: 57%|█████▋ | 178/311 [00:00<00:00, 345.22it/s, Materializing param=model.layers.15.self_attn.v_proj.weight] Loading weights: 57%|█████▋ | 178/311 [00:00<00:00, 313.51it/s, Materializing param=model.layers.15.self_attn.v_proj.weight] Loading weights: 52%|█████▏ | 162/311 [00:00<00:00, 290.52it/s, Materializing param=model.layers.14.self_attn.k_norm.weight] Loading weights: 57%|█████▋ | 178/311 [00:00<00:00, 345.22it/s, Materializing param=model.layers.15.self_attn.v_proj.weight] Loading weights: 52%|█████▏ | 162/311 [00:00<00:00, 290.52it/s, Materializing param=model.layers.14.self_attn.k_norm.weight] Loading weights: 52%|█████▏ | 163/311 [00:00<00:00, 290.52it/s, Materializing param=model.layers.14.self_attn.k_proj.weight] Loading weights: 52%|█████▏ | 163/311 [00:00<00:00, 290.52it/s, Materializing param=model.layers.14.self_attn.k_proj.weight] Loading weights: 58%|█████▊ | 179/311 [00:00<00:00, 313.51it/s, Materializing param=model.layers.16.input_layernorm.weight] Loading weights: 58%|█████▊ | 179/311 [00:00<00:00, 345.22it/s, Materializing param=model.layers.16.input_layernorm.weight] Loading weights: 58%|█████▊ | 179/311 [00:00<00:00, 313.51it/s, Materializing param=model.layers.16.input_layernorm.weight] Loading weights: 58%|█████▊ | 179/311 [00:00<00:00, 345.22it/s, Materializing param=model.layers.16.input_layernorm.weight] Loading weights: 53%|█████▎ | 164/311 [00:00<00:00, 290.52it/s, Materializing param=model.layers.14.self_attn.o_proj.weight] Loading weights: 53%|█████▎ | 164/311 [00:00<00:00, 290.52it/s, Materializing param=model.layers.14.self_attn.o_proj.weight] Loading weights: 27%|██▋ | 84/311 [00:00<00:01, 216.05it/s, Materializing param=model.layers.7.post_attention_layernorm.weight] Loading weights: 27%|██▋ | 84/311 [00:00<00:01, 216.05it/s, Materializing param=model.layers.7.post_attention_layernorm.weight] Loading weights: 27%|██▋ | 85/311 [00:00<00:01, 216.05it/s, Materializing param=model.layers.7.self_attn.k_norm.weight] Loading weights: 27%|██▋ | 85/311 [00:00<00:01, 216.05it/s, Materializing param=model.layers.7.self_attn.k_norm.weight] Loading weights: 28%|██▊ | 86/311 [00:00<00:01, 216.05it/s, Materializing param=model.layers.7.self_attn.k_proj.weight] Loading weights: 28%|██▊ | 86/311 [00:00<00:01, 216.05it/s, Materializing param=model.layers.7.self_attn.k_proj.weight] Loading weights: 37%|███▋ | 115/311 [00:00<00:00, 200.31it/s, Materializing param=model.layers.10.mlp.gate_proj.weight] Loading weights: 37%|███▋ | 115/311 [00:00<00:00, 200.31it/s, Materializing param=model.layers.10.mlp.gate_proj.weight] Loading weights: 28%|██▊ | 87/311 [00:00<00:01, 216.05it/s, Materializing param=model.layers.7.self_attn.o_proj.weight] Loading weights: 53%|█████▎ | 165/311 [00:00<00:00, 290.52it/s, Materializing param=model.layers.14.self_attn.q_norm.weight] Loading weights: 52%|█████▏ | 161/311 [00:00<00:00, 315.95it/s, Materializing param=model.layers.14.post_attention_layernorm.weight] Loading weights: 28%|██▊ | 87/311 [00:00<00:01, 216.05it/s, Materializing param=model.layers.7.self_attn.o_proj.weight] Loading weights: 53%|█████▎ | 165/311 [00:00<00:00, 290.52it/s, Materializing param=model.layers.14.self_attn.q_norm.weight] Loading weights: 52%|█████▏ | 161/311 [00:00<00:00, 315.95it/s, Materializing param=model.layers.14.post_attention_layernorm.weight] Loading weights: 52%|█████▏ | 162/311 [00:00<00:00, 315.95it/s, Materializing param=model.layers.14.self_attn.k_norm.weight] Loading weights: 53%|█████▎ | 166/311 [00:00<00:00, 290.52it/s, Materializing param=model.layers.14.self_attn.q_proj.weight] Loading weights: 52%|█████▏ | 162/311 [00:00<00:00, 315.95it/s, Materializing param=model.layers.14.self_attn.k_norm.weight] Loading weights: 53%|█████▎ | 166/311 [00:00<00:00, 290.52it/s, Materializing param=model.layers.14.self_attn.q_proj.weight] Loading weights: 54%|█████▎ | 167/311 [00:00<00:00, 290.52it/s, Materializing param=model.layers.14.self_attn.v_proj.weight] Loading weights: 41%|████ | 128/311 [00:00<00:00, 240.68it/s, Materializing param=model.layers.11.post_attention_layernorm.weight] Loading weights: 52%|█████▏ | 163/311 [00:00<00:00, 315.95it/s, Materializing param=model.layers.14.self_attn.k_proj.weight] Loading weights: 41%|████ | 128/311 [00:00<00:00, 240.68it/s, Materializing param=model.layers.11.post_attention_layernorm.weight] Loading weights: 54%|█████▎ | 167/311 [00:00<00:00, 290.52it/s, Materializing param=model.layers.14.self_attn.v_proj.weight] Loading weights: 52%|█████▏ | 163/311 [00:00<00:00, 315.95it/s, Materializing param=model.layers.14.self_attn.k_proj.weight] Loading weights: 53%|█████▎ | 164/311 [00:00<00:00, 315.95it/s, Materializing param=model.layers.14.self_attn.o_proj.weight] Loading weights: 53%|█████▎ | 164/311 [00:00<00:00, 315.95it/s, Materializing param=model.layers.14.self_attn.o_proj.weight] Loading weights: 41%|████▏ | 129/311 [00:00<00:00, 240.68it/s, Materializing param=model.layers.11.self_attn.k_norm.weight] Loading weights: 41%|████▏ | 129/311 [00:00<00:00, 240.68it/s, Materializing param=model.layers.11.self_attn.k_norm.weight] Loading weights: 42%|████▏ | 130/311 [00:00<00:00, 240.68it/s, Materializing param=model.layers.11.self_attn.k_proj.weight] Loading weights: 42%|████▏ | 130/311 [00:00<00:00, 240.68it/s, Materializing param=model.layers.11.self_attn.k_proj.weight] Loading weights: 54%|█████▍ | 168/311 [00:00<00:00, 290.52it/s, Materializing param=model.layers.15.input_layernorm.weight] Loading weights: 54%|█████▍ | 168/311 [00:00<00:00, 290.52it/s, Materializing param=model.layers.15.input_layernorm.weight] Loading weights: 58%|█████▊ | 180/311 [00:00<00:00, 313.51it/s, Materializing param=model.layers.16.mlp.down_proj.weight] Loading weights: 58%|█████▊ | 180/311 [00:00<00:00, 313.51it/s, Materializing param=model.layers.16.mlp.down_proj.weight] Loading weights: 54%|█████▍ | 169/311 [00:00<00:00, 290.52it/s, Materializing param=model.layers.15.mlp.down_proj.weight] Loading weights: 42%|████▏ | 131/311 [00:00<00:00, 240.68it/s, Materializing param=model.layers.11.self_attn.o_proj.weight] Loading weights: 54%|█████▍ | 169/311 [00:00<00:00, 290.52it/s, Materializing param=model.layers.15.mlp.down_proj.weight] Loading weights: 42%|████▏ | 131/311 [00:00<00:00, 240.68it/s, Materializing param=model.layers.11.self_attn.o_proj.weight] Loading weights: 58%|█████▊ | 181/311 [00:00<00:00, 313.51it/s, Materializing param=model.layers.16.mlp.gate_proj.weight] Loading weights: 58%|█████▊ | 180/311 [00:00<00:00, 345.22it/s, Materializing param=model.layers.16.mlp.down_proj.weight] Loading weights: 58%|█████▊ | 181/311 [00:00<00:00, 313.51it/s, Materializing param=model.layers.16.mlp.gate_proj.weight] Loading weights: 58%|█████▊ | 180/311 [00:00<00:00, 345.22it/s, Materializing param=model.layers.16.mlp.down_proj.weight] Loading weights: 28%|██▊ | 88/311 [00:00<00:01, 216.05it/s, Materializing param=model.layers.7.self_attn.q_norm.weight] Loading weights: 28%|██▊ | 88/311 [00:00<00:01, 216.05it/s, Materializing param=model.layers.7.self_attn.q_norm.weight] Loading weights: 29%|██▊ | 89/311 [00:00<00:01, 216.05it/s, Materializing param=model.layers.7.self_attn.q_proj.weight] Loading weights: 29%|██▊ | 89/311 [00:00<00:01, 216.05it/s, Materializing param=model.layers.7.self_attn.q_proj.weight] Loading weights: 53%|█████▎ | 165/311 [00:00<00:00, 315.95it/s, Materializing param=model.layers.14.self_attn.q_norm.weight] Loading weights: 53%|█████▎ | 165/311 [00:00<00:00, 315.95it/s, Materializing param=model.layers.14.self_attn.q_norm.weight] Loading weights: 53%|█████▎ | 166/311 [00:00<00:00, 315.95it/s, Materializing param=model.layers.14.self_attn.q_proj.weight] Loading weights: 53%|█████▎ | 166/311 [00:00<00:00, 315.95it/s, Materializing param=model.layers.14.self_attn.q_proj.weight] Loading weights: 29%|██▉ | 90/311 [00:00<00:01, 216.05it/s, Materializing param=model.layers.7.self_attn.v_proj.weight] Loading weights: 29%|██▉ | 90/311 [00:00<00:01, 216.05it/s, Materializing param=model.layers.7.self_attn.v_proj.weight] Loading weights: 54%|█████▎ | 167/311 [00:00<00:00, 315.95it/s, Materializing param=model.layers.14.self_attn.v_proj.weight] Loading weights: 54%|█████▎ | 167/311 [00:00<00:00, 315.95it/s, Materializing param=model.layers.14.self_attn.v_proj.weight] Loading weights: 59%|█████▊ | 182/311 [00:00<00:00, 348.61it/s, Materializing param=model.layers.16.mlp.gate_proj.weight] Loading weights: 59%|█████▊ | 182/311 [00:00<00:00, 348.61it/s, Materializing param=model.layers.16.mlp.up_proj.weight] Loading weights: 59%|█████▊ | 182/311 [00:00<00:00, 348.61it/s, Materializing param=model.layers.16.mlp.up_proj.weight] Loading weights: 29%|██▉ | 91/311 [00:00<00:01, 216.05it/s, Materializing param=model.layers.8.input_layernorm.weight] Loading weights: 54%|█████▍ | 168/311 [00:00<00:00, 315.95it/s, Materializing param=model.layers.15.input_layernorm.weight] Loading weights: 29%|██▉ | 91/311 [00:00<00:01, 216.05it/s, Materializing param=model.layers.8.input_layernorm.weight] Loading weights: 54%|█████▍ | 168/311 [00:00<00:00, 315.95it/s, Materializing param=model.layers.15.input_layernorm.weight] Loading weights: 54%|█████▍ | 169/311 [00:00<00:00, 315.95it/s, Materializing param=model.layers.15.mlp.down_proj.weight] Loading weights: 58%|█████▊ | 181/311 [00:00<00:00, 345.22it/s, Materializing param=model.layers.16.mlp.gate_proj.weight] Loading weights: 54%|█████▍ | 169/311 [00:00<00:00, 315.95it/s, Materializing param=model.layers.15.mlp.down_proj.weight] Loading weights: 58%|█████▊ | 181/311 [00:00<00:00, 345.22it/s, Materializing param=model.layers.16.mlp.gate_proj.weight] Loading weights: 59%|█████▊ | 182/311 [00:00<00:00, 345.22it/s, Materializing param=model.layers.16.mlp.up_proj.weight] Loading weights: 59%|█████▊ | 182/311 [00:00<00:00, 345.22it/s, Materializing param=model.layers.16.mlp.up_proj.weight] Loading weights: 55%|█████▍ | 170/311 [00:00<00:00, 290.52it/s, Materializing param=model.layers.15.mlp.gate_proj.weight] Loading weights: 55%|█████▍ | 170/311 [00:00<00:00, 290.52it/s, Materializing param=model.layers.15.mlp.gate_proj.weight] Loading weights: 37%|███▋ | 116/311 [00:00<00:00, 255.40it/s, Materializing param=model.layers.10.mlp.gate_proj.weight] Loading weights: 42%|████▏ | 132/311 [00:00<00:00, 240.68it/s, Materializing param=model.layers.11.self_attn.q_norm.weight] Loading weights: 37%|███▋ | 116/311 [00:00<00:00, 255.40it/s, Materializing param=model.layers.10.mlp.up_proj.weight] Loading weights: 42%|████▏ | 132/311 [00:00<00:00, 240.68it/s, Materializing param=model.layers.11.self_attn.q_norm.weight] Loading weights: 37%|███▋ | 116/311 [00:00<00:00, 255.40it/s, Materializing param=model.layers.10.mlp.up_proj.weight] Loading weights: 43%|████▎ | 133/311 [00:00<00:00, 240.68it/s, Materializing param=model.layers.11.self_attn.q_proj.weight] Loading weights: 43%|████▎ | 133/311 [00:00<00:00, 240.68it/s, Materializing param=model.layers.11.self_attn.q_proj.weight] Loading weights: 43%|████▎ | 134/311 [00:00<00:00, 240.68it/s, Materializing param=model.layers.11.self_attn.v_proj.weight] Loading weights: 43%|████▎ | 134/311 [00:00<00:00, 240.68it/s, Materializing param=model.layers.11.self_attn.v_proj.weight] Loading weights: 43%|████▎ | 135/311 [00:00<00:00, 240.68it/s, Materializing param=model.layers.12.input_layernorm.weight] Loading weights: 43%|████▎ | 135/311 [00:00<00:00, 240.68it/s, Materializing param=model.layers.12.input_layernorm.weight] Loading weights: 44%|████▎ | 136/311 [00:00<00:00, 240.68it/s, Materializing param=model.layers.12.mlp.down_proj.weight] Loading weights: 44%|████▎ | 136/311 [00:00<00:00, 240.68it/s, Materializing param=model.layers.12.mlp.down_proj.weight] Loading weights: 59%|█████▉ | 183/311 [00:00<00:00, 348.61it/s, Materializing param=model.layers.16.post_attention_layernorm.weight] Loading weights: 59%|█████▉ | 183/311 [00:00<00:00, 348.61it/s, Materializing param=model.layers.16.post_attention_layernorm.weight] Loading weights: 59%|█████▉ | 183/311 [00:00<00:00, 345.22it/s, Materializing param=model.layers.16.post_attention_layernorm.weight] Loading weights: 59%|█████▉ | 184/311 [00:00<00:00, 348.61it/s, Materializing param=model.layers.16.self_attn.k_norm.weight] Loading weights: 59%|█████▉ | 183/311 [00:00<00:00, 345.22it/s, Materializing param=model.layers.16.post_attention_layernorm.weight] Loading weights: 59%|█████▉ | 184/311 [00:00<00:00, 348.61it/s, Materializing param=model.layers.16.self_attn.k_norm.weight] Loading weights: 59%|█████▉ | 184/311 [00:00<00:00, 345.22it/s, Materializing param=model.layers.16.self_attn.k_norm.weight] Loading weights: 59%|█████▉ | 185/311 [00:00<00:00, 348.61it/s, Materializing param=model.layers.16.self_attn.k_proj.weight] Loading weights: 59%|█████▉ | 184/311 [00:00<00:00, 345.22it/s, Materializing param=model.layers.16.self_attn.k_norm.weight] Loading weights: 59%|█████▉ | 185/311 [00:00<00:00, 348.61it/s, Materializing param=model.layers.16.self_attn.k_proj.weight] Loading weights: 59%|█████▉ | 185/311 [00:00<00:00, 345.22it/s, Materializing param=model.layers.16.self_attn.k_proj.weight] Loading weights: 59%|█████▉ | 185/311 [00:00<00:00, 345.22it/s, Materializing param=model.layers.16.self_attn.k_proj.weight] Loading weights: 60%|█████▉ | 186/311 [00:00<00:00, 348.61it/s, Materializing param=model.layers.16.self_attn.o_proj.weight] Loading weights: 60%|█████▉ | 186/311 [00:00<00:00, 345.22it/s, Materializing param=model.layers.16.self_attn.o_proj.weight] Loading weights: 60%|█████▉ | 186/311 [00:00<00:00, 348.61it/s, Materializing param=model.layers.16.self_attn.o_proj.weight] Loading weights: 60%|█████▉ | 186/311 [00:00<00:00, 345.22it/s, Materializing param=model.layers.16.self_attn.o_proj.weight] Loading weights: 55%|█████▍ | 170/311 [00:00<00:00, 315.95it/s, Materializing param=model.layers.15.mlp.gate_proj.weight] Loading weights: 30%|██▉ | 92/311 [00:00<00:01, 216.05it/s, Materializing param=model.layers.8.mlp.down_proj.weight] Loading weights: 55%|█████▍ | 170/311 [00:00<00:00, 315.95it/s, Materializing param=model.layers.15.mlp.gate_proj.weight] Loading weights: 30%|██▉ | 92/311 [00:00<00:01, 216.05it/s, Materializing param=model.layers.8.mlp.down_proj.weight] Loading weights: 55%|█████▍ | 171/311 [00:00<00:00, 290.52it/s, Materializing param=model.layers.15.mlp.up_proj.weight] Loading weights: 30%|██▉ | 93/311 [00:00<00:01, 216.05it/s, Materializing param=model.layers.8.mlp.gate_proj.weight] Loading weights: 30%|██▉ | 93/311 [00:00<00:01, 216.05it/s, Materializing param=model.layers.8.mlp.gate_proj.weight] Loading weights: 55%|█████▍ | 171/311 [00:00<00:00, 290.52it/s, Materializing param=model.layers.15.mlp.up_proj.weight] Loading weights: 60%|██████ | 187/311 [00:00<00:00, 345.22it/s, Materializing param=model.layers.16.self_attn.q_norm.weight] Loading weights: 60%|██████ | 187/311 [00:00<00:00, 348.61it/s, Materializing param=model.layers.16.self_attn.q_norm.weight] Loading weights: 60%|██████ | 187/311 [00:00<00:00, 345.22it/s, Materializing param=model.layers.16.self_attn.q_norm.weight] Loading weights: 60%|██████ | 187/311 [00:00<00:00, 348.61it/s, Materializing param=model.layers.16.self_attn.q_norm.weight] Loading weights: 60%|██████ | 188/311 [00:00<00:00, 345.22it/s, Materializing param=model.layers.16.self_attn.q_proj.weight] Loading weights: 60%|██████ | 188/311 [00:00<00:00, 348.61it/s, Materializing param=model.layers.16.self_attn.q_proj.weight] Loading weights: 60%|██████ | 188/311 [00:00<00:00, 345.22it/s, Materializing param=model.layers.16.self_attn.q_proj.weight] Loading weights: 38%|███▊ | 117/311 [00:00<00:00, 255.40it/s, Materializing param=model.layers.10.post_attention_layernorm.weight] Loading weights: 60%|██████ | 188/311 [00:00<00:00, 348.61it/s, Materializing param=model.layers.16.self_attn.q_proj.weight] Loading weights: 38%|███▊ | 117/311 [00:00<00:00, 255.40it/s, Materializing param=model.layers.10.post_attention_layernorm.weight] Loading weights: 38%|███▊ | 118/311 [00:00<00:00, 255.40it/s, Materializing param=model.layers.10.self_attn.k_norm.weight] Loading weights: 38%|███▊ | 118/311 [00:00<00:00, 255.40it/s, Materializing param=model.layers.10.self_attn.k_norm.weight] Loading weights: 44%|████▍ | 137/311 [00:00<00:00, 240.68it/s, Materializing param=model.layers.12.mlp.gate_proj.weight] Loading weights: 38%|███▊ | 119/311 [00:00<00:00, 255.40it/s, Materializing param=model.layers.10.self_attn.k_proj.weight] Loading weights: 44%|████▍ | 137/311 [00:00<00:00, 240.68it/s, Materializing param=model.layers.12.mlp.gate_proj.weight] Loading weights: 38%|███▊ | 119/311 [00:00<00:00, 255.40it/s, Materializing param=model.layers.10.self_attn.k_proj.weight] Loading weights: 61%|██████ | 189/311 [00:00<00:00, 348.61it/s, Materializing param=model.layers.16.self_attn.v_proj.weight] Loading weights: 61%|██████ | 189/311 [00:00<00:00, 348.61it/s, Materializing param=model.layers.16.self_attn.v_proj.weight] Loading weights: 61%|██████ | 189/311 [00:00<00:00, 345.22it/s, Materializing param=model.layers.16.self_attn.v_proj.weight] Loading weights: 61%|██████ | 189/311 [00:00<00:00, 345.22it/s, Materializing param=model.layers.16.self_attn.v_proj.weight] Loading weights: 39%|███▊ | 120/311 [00:00<00:00, 255.40it/s, Materializing param=model.layers.10.self_attn.o_proj.weight] Loading weights: 39%|███▊ | 120/311 [00:00<00:00, 255.40it/s, Materializing param=model.layers.10.self_attn.o_proj.weight] Loading weights: 61%|██████ | 190/311 [00:00<00:00, 345.22it/s, Materializing param=model.layers.17.input_layernorm.weight] Loading weights: 61%|██████ | 190/311 [00:00<00:00, 345.22it/s, Materializing param=model.layers.17.input_layernorm.weight] Loading weights: 61%|██████ | 190/311 [00:00<00:00, 348.61it/s, Materializing param=model.layers.17.input_layernorm.weight] Loading weights: 61%|██████ | 190/311 [00:00<00:00, 348.61it/s, Materializing param=model.layers.17.input_layernorm.weight] Loading weights: 61%|██████▏ | 191/311 [00:00<00:00, 348.61it/s, Materializing param=model.layers.17.mlp.down_proj.weight] Loading weights: 61%|██████▏ | 191/311 [00:00<00:00, 348.61it/s, Materializing param=model.layers.17.mlp.down_proj.weight] Loading weights: 61%|██████▏ | 191/311 [00:00<00:00, 345.22it/s, Materializing param=model.layers.17.mlp.down_proj.weight] Loading weights: 61%|██████▏ | 191/311 [00:00<00:00, 345.22it/s, Materializing param=model.layers.17.mlp.down_proj.weight] Loading weights: 39%|███▉ | 121/311 [00:00<00:00, 255.40it/s, Materializing param=model.layers.10.self_attn.q_norm.weight] Loading weights: 39%|███▉ | 121/311 [00:00<00:00, 255.40it/s, Materializing param=model.layers.10.self_attn.q_norm.weight] Loading weights: 55%|█████▍ | 171/311 [00:00<00:00, 315.95it/s, Materializing param=model.layers.15.mlp.up_proj.weight] Loading weights: 55%|█████▍ | 171/311 [00:00<00:00, 315.95it/s, Materializing param=model.layers.15.mlp.up_proj.weight] Loading weights: 30%|███ | 94/311 [00:00<00:01, 216.05it/s, Materializing param=model.layers.8.mlp.up_proj.weight] Loading weights: 30%|███ | 94/311 [00:00<00:01, 216.05it/s, Materializing param=model.layers.8.mlp.up_proj.weight] Loading weights: 55%|█████▌ | 172/311 [00:00<00:00, 331.54it/s, Materializing param=model.layers.15.mlp.up_proj.weight] Loading weights: 55%|█████▌ | 172/311 [00:00<00:00, 331.54it/s, Materializing param=model.layers.15.post_attention_layernorm.weight] Loading weights: 55%|█████▌ | 172/311 [00:00<00:00, 331.54it/s, Materializing param=model.layers.15.post_attention_layernorm.weight] Loading weights: 56%|█████▌ | 173/311 [00:00<00:00, 331.54it/s, Materializing param=model.layers.15.self_attn.k_norm.weight] Loading weights: 56%|█████▌ | 173/311 [00:00<00:00, 331.54it/s, Materializing param=model.layers.15.self_attn.k_norm.weight] Loading weights: 56%|█████▌ | 174/311 [00:00<00:00, 331.54it/s, Materializing param=model.layers.15.self_attn.k_proj.weight] Loading weights: 56%|█████▌ | 174/311 [00:00<00:00, 331.54it/s, Materializing param=model.layers.15.self_attn.k_proj.weight] Loading weights: 56%|█████▋ | 175/311 [00:00<00:00, 331.54it/s, Materializing param=model.layers.15.self_attn.o_proj.weight] Loading weights: 39%|███▉ | 122/311 [00:00<00:00, 255.40it/s, Materializing param=model.layers.10.self_attn.q_proj.weight] Loading weights: 56%|█████▋ | 175/311 [00:00<00:00, 331.54it/s, Materializing param=model.layers.15.self_attn.o_proj.weight] Loading weights: 39%|███▉ | 122/311 [00:00<00:00, 255.40it/s, Materializing param=model.layers.10.self_attn.q_proj.weight] Loading weights: 40%|███▉ | 123/311 [00:00<00:00, 255.40it/s, Materializing param=model.layers.10.self_attn.v_proj.weight] Loading weights: 40%|███▉ | 123/311 [00:00<00:00, 255.40it/s, Materializing param=model.layers.10.self_attn.v_proj.weight][WARNING|2026-04-29 17:06:08] llamafactory.train.callbacks:155 >> Previous trainer log in this folder will be deleted. + Loading weights: 40%|███▉ | 124/311 [00:00<00:00, 255.40it/s, Materializing param=model.layers.11.input_layernorm.weight] Loading weights: 40%|███▉ | 124/311 [00:00<00:00, 255.40it/s, Materializing param=model.layers.11.input_layernorm.weight] Loading weights: 40%|████ | 125/311 [00:00<00:00, 255.40it/s, Materializing param=model.layers.11.mlp.down_proj.weight] Loading weights: 40%|████ | 125/311 [00:00<00:00, 255.40it/s, Materializing param=model.layers.11.mlp.down_proj.weight] Loading weights: 44%|████▍ | 138/311 [00:00<00:00, 240.68it/s, Materializing param=model.layers.12.mlp.up_proj.weight] Loading weights: 44%|████▍ | 138/311 [00:00<00:00, 240.68it/s, Materializing param=model.layers.12.mlp.up_proj.weight][WARNING|trainer_utils.py:1234] 2026-04-29 17:06:08,234 >> The tokenizer has new PAD/BOS/EOS tokens that differ from the model config and generation config. The model config and generation config were aligned accordingly, being updated with the tokenizer's values. Updated tokens: {'bos_token_id': None, 'pad_token_id': 151643}. + Loading weights: 57%|█████▋ | 176/311 [00:00<00:00, 331.54it/s, Materializing param=model.layers.15.self_attn.q_norm.weight] Loading weights: 57%|█████▋ | 176/311 [00:00<00:00, 331.54it/s, Materializing param=model.layers.15.self_attn.q_norm.weight] Loading weights: 62%|██████▏ | 192/311 [00:00<00:00, 345.22it/s, Materializing param=model.layers.17.mlp.gate_proj.weight] Loading weights: 62%|██████▏ | 192/311 [00:00<00:00, 348.61it/s, Materializing param=model.layers.17.mlp.gate_proj.weight] Loading weights: 62%|██████▏ | 192/311 [00:00<00:00, 345.22it/s, Materializing param=model.layers.17.mlp.gate_proj.weight] Loading weights: 62%|██████▏ | 192/311 [00:00<00:00, 348.61it/s, Materializing param=model.layers.17.mlp.gate_proj.weight] Loading weights: 55%|█████▌ | 172/311 [00:00<00:00, 315.95it/s, Materializing param=model.layers.15.post_attention_layernorm.weight] Loading weights: 55%|█████▌ | 172/311 [00:00<00:00, 315.95it/s, Materializing param=model.layers.15.post_attention_layernorm.weight] Loading weights: 57%|█████▋ | 177/311 [00:00<00:00, 331.54it/s, Materializing param=model.layers.15.self_attn.q_proj.weight] Loading weights: 56%|█████▌ | 173/311 [00:00<00:00, 315.95it/s, Materializing param=model.layers.15.self_attn.k_norm.weight] Loading weights: 57%|█████▋ | 177/311 [00:00<00:00, 331.54it/s, Materializing param=model.layers.15.self_attn.q_proj.weight] Loading weights: 56%|█████▌ | 173/311 [00:00<00:00, 315.95it/s, Materializing param=model.layers.15.self_attn.k_norm.weight] Loading weights: 57%|█████▋ | 178/311 [00:00<00:00, 331.54it/s, Materializing param=model.layers.15.self_attn.v_proj.weight] Loading weights: 57%|█████▋ | 178/311 [00:00<00:00, 331.54it/s, Materializing param=model.layers.15.self_attn.v_proj.weight] Loading weights: 56%|█████▌ | 174/311 [00:00<00:00, 315.95it/s, Materializing param=model.layers.15.self_attn.k_proj.weight] Loading weights: 56%|█████▌ | 174/311 [00:00<00:00, 315.95it/s, Materializing param=model.layers.15.self_attn.k_proj.weight] Loading weights: 56%|█████▋ | 175/311 [00:00<00:00, 315.95it/s, Materializing param=model.layers.15.self_attn.o_proj.weight] Loading weights: 56%|█████▋ | 175/311 [00:00<00:00, 315.95it/s, Materializing param=model.layers.15.self_attn.o_proj.weight] Loading weights: 58%|█████▊ | 179/311 [00:00<00:00, 331.54it/s, Materializing param=model.layers.16.input_layernorm.weight] Loading weights: 58%|█████▊ | 179/311 [00:00<00:00, 331.54it/s, Materializing param=model.layers.16.input_layernorm.weight] Loading weights: 31%|███ | 95/311 [00:00<00:00, 216.05it/s, Materializing param=model.layers.8.post_attention_layernorm.weight] Loading weights: 58%|█████▊ | 180/311 [00:00<00:00, 331.54it/s, Materializing param=model.layers.16.mlp.down_proj.weight] Loading weights: 31%|███ | 95/311 [00:00<00:00, 216.05it/s, Materializing param=model.layers.8.post_attention_layernorm.weight] Loading weights: 58%|█████▊ | 180/311 [00:00<00:00, 331.54it/s, Materializing param=model.layers.16.mlp.down_proj.weight] Loading weights: 31%|███ | 96/311 [00:00<00:00, 216.05it/s, Materializing param=model.layers.8.self_attn.k_norm.weight] Loading weights: 31%|███ | 96/311 [00:00<00:00, 216.05it/s, Materializing param=model.layers.8.self_attn.k_norm.weight] Loading weights: 31%|███ | 97/311 [00:00<00:00, 216.05it/s, Materializing param=model.layers.8.self_attn.k_proj.weight] Loading weights: 31%|███ | 97/311 [00:00<00:00, 216.05it/s, Materializing param=model.layers.8.self_attn.k_proj.weight] Loading weights: 57%|█████▋ | 176/311 [00:00<00:00, 315.95it/s, Materializing param=model.layers.15.self_attn.q_norm.weight] Loading weights: 57%|█████▋ | 176/311 [00:00<00:00, 315.95it/s, Materializing param=model.layers.15.self_attn.q_norm.weight] Loading weights: 32%|███▏ | 98/311 [00:00<00:00, 216.05it/s, Materializing param=model.layers.8.self_attn.o_proj.weight] Loading weights: 32%|███▏ | 98/311 [00:00<00:00, 216.05it/s, Materializing param=model.layers.8.self_attn.o_proj.weight] Loading weights: 57%|█████▋ | 177/311 [00:00<00:00, 315.95it/s, Materializing param=model.layers.15.self_attn.q_proj.weight] Loading weights: 57%|█████▋ | 177/311 [00:00<00:00, 315.95it/s, Materializing param=model.layers.15.self_attn.q_proj.weight] Loading weights: 62%|██████▏ | 193/311 [00:00<00:00, 348.61it/s, Materializing param=model.layers.17.mlp.up_proj.weight] Loading weights: 62%|██████▏ | 193/311 [00:00<00:00, 345.22it/s, Materializing param=model.layers.17.mlp.up_proj.weight] Loading weights: 62%|██████▏ | 193/311 [00:00<00:00, 348.61it/s, Materializing param=model.layers.17.mlp.up_proj.weight] Loading weights: 62%|██████▏ | 193/311 [00:00<00:00, 345.22it/s, Materializing param=model.layers.17.mlp.up_proj.weight] Loading weights: 45%|████▍ | 139/311 [00:00<00:00, 240.68it/s, Materializing param=model.layers.12.post_attention_layernorm.weight] Loading weights: 45%|████▍ | 139/311 [00:00<00:00, 240.68it/s, Materializing param=model.layers.12.post_attention_layernorm.weight] Loading weights: 32%|███▏ | 99/311 [00:00<00:00, 216.05it/s, Materializing param=model.layers.8.self_attn.q_norm.weight] Loading weights: 32%|███▏ | 99/311 [00:00<00:00, 216.05it/s, Materializing param=model.layers.8.self_attn.q_norm.weight] Loading weights: 32%|███▏ | 100/311 [00:00<00:00, 216.05it/s, Materializing param=model.layers.8.self_attn.q_proj.weight] Loading weights: 45%|████▌ | 140/311 [00:00<00:00, 240.68it/s, Materializing param=model.layers.12.self_attn.k_norm.weight] Loading weights: 32%|███▏ | 100/311 [00:00<00:00, 216.05it/s, Materializing param=model.layers.8.self_attn.q_proj.weight] Loading weights: 45%|████▌ | 140/311 [00:00<00:00, 240.68it/s, Materializing param=model.layers.12.self_attn.k_norm.weight] Loading weights: 45%|████▌ | 141/311 [00:00<00:00, 240.68it/s, Materializing param=model.layers.12.self_attn.k_proj.weight] Loading weights: 45%|████▌ | 141/311 [00:00<00:00, 240.68it/s, Materializing param=model.layers.12.self_attn.k_proj.weight] Loading weights: 46%|████▌ | 142/311 [00:00<00:00, 240.68it/s, Materializing param=model.layers.12.self_attn.o_proj.weight] Loading weights: 46%|████▌ | 142/311 [00:00<00:00, 240.68it/s, Materializing param=model.layers.12.self_attn.o_proj.weight] Loading weights: 57%|█████▋ | 178/311 [00:00<00:00, 315.95it/s, Materializing param=model.layers.15.self_attn.v_proj.weight] Loading weights: 57%|█████▋ | 178/311 [00:00<00:00, 315.95it/s, Materializing param=model.layers.15.self_attn.v_proj.weight] Loading weights: 58%|█████▊ | 179/311 [00:00<00:00, 315.95it/s, Materializing param=model.layers.16.input_layernorm.weight] Loading weights: 58%|█████▊ | 179/311 [00:00<00:00, 315.95it/s, Materializing param=model.layers.16.input_layernorm.weight] Loading weights: 58%|█████▊ | 180/311 [00:00<00:00, 315.95it/s, Materializing param=model.layers.16.mlp.down_proj.weight] Loading weights: 58%|█████▊ | 180/311 [00:00<00:00, 315.95it/s, Materializing param=model.layers.16.mlp.down_proj.weight] Loading weights: 32%|███▏ | 101/311 [00:00<00:00, 216.05it/s, Materializing param=model.layers.8.self_attn.v_proj.weight] Loading weights: 32%|███▏ | 101/311 [00:00<00:00, 216.05it/s, Materializing param=model.layers.8.self_attn.v_proj.weight] Loading weights: 58%|█████▊ | 181/311 [00:00<00:00, 331.54it/s, Materializing param=model.layers.16.mlp.gate_proj.weight] Loading weights: 58%|█████▊ | 181/311 [00:00<00:00, 331.54it/s, Materializing param=model.layers.16.mlp.gate_proj.weight] Loading weights: 46%|████▌ | 143/311 [00:00<00:00, 240.68it/s, Materializing param=model.layers.12.self_attn.q_norm.weight] Loading weights: 46%|████▌ | 143/311 [00:00<00:00, 240.68it/s, Materializing param=model.layers.12.self_attn.q_norm.weight] Loading weights: 33%|███▎ | 102/311 [00:00<00:00, 216.05it/s, Materializing param=model.layers.9.input_layernorm.weight] Loading weights: 33%|███▎ | 102/311 [00:00<00:00, 216.05it/s, Materializing param=model.layers.9.input_layernorm.weight] Loading weights: 33%|███▎ | 103/311 [00:00<00:00, 216.05it/s, Materializing param=model.layers.9.mlp.down_proj.weight] Loading weights: 33%|███▎ | 103/311 [00:00<00:00, 216.05it/s, Materializing param=model.layers.9.mlp.down_proj.weight] Loading weights: 46%|████▋ | 144/311 [00:00<00:00, 240.68it/s, Materializing param=model.layers.12.self_attn.q_proj.weight] Loading weights: 62%|██████▏ | 194/311 [00:00<00:00, 372.07it/s, Materializing param=model.layers.17.mlp.up_proj.weight] Loading weights: 62%|██████▏ | 194/311 [00:00<00:00, 372.07it/s, Materializing param=model.layers.17.post_attention_layernorm.weight] Loading weights: 62%|██████▏ | 194/311 [00:00<00:00, 348.61it/s, Materializing param=model.layers.17.post_attention_layernorm.weight] Loading weights: 46%|████▋ | 144/311 [00:00<00:00, 240.68it/s, Materializing param=model.layers.12.self_attn.q_proj.weight] Loading weights: 62%|██████▏ | 194/311 [00:00<00:00, 372.07it/s, Materializing param=model.layers.17.post_attention_layernorm.weight] Loading weights: 62%|██████▏ | 194/311 [00:00<00:00, 348.61it/s, Materializing param=model.layers.17.post_attention_layernorm.weight] Loading weights: 47%|████▋ | 145/311 [00:00<00:00, 240.68it/s, Materializing param=model.layers.12.self_attn.v_proj.weight] Loading weights: 47%|████▋ | 145/311 [00:00<00:00, 240.68it/s, Materializing param=model.layers.12.self_attn.v_proj.weight] Loading weights: 63%|██████▎ | 195/311 [00:00<00:00, 348.61it/s, Materializing param=model.layers.17.self_attn.k_norm.weight] Loading weights: 63%|██████▎ | 195/311 [00:00<00:00, 348.61it/s, Materializing param=model.layers.17.self_attn.k_norm.weight] Loading weights: 63%|██████▎ | 195/311 [00:00<00:00, 372.07it/s, Materializing param=model.layers.17.self_attn.k_norm.weight] Loading weights: 63%|██████▎ | 195/311 [00:00<00:00, 372.07it/s, Materializing param=model.layers.17.self_attn.k_norm.weight] Loading weights: 41%|████ | 126/311 [00:00<00:00, 255.40it/s, Materializing param=model.layers.11.mlp.gate_proj.weight] Loading weights: 63%|██████▎ | 196/311 [00:00<00:00, 348.61it/s, Materializing param=model.layers.17.self_attn.k_proj.weight] Loading weights: 41%|████ | 126/311 [00:00<00:00, 255.40it/s, Materializing param=model.layers.11.mlp.gate_proj.weight] Loading weights: 63%|██████▎ | 196/311 [00:00<00:00, 348.61it/s, Materializing param=model.layers.17.self_attn.k_proj.weight] Loading weights: 63%|██████▎ | 196/311 [00:00<00:00, 372.07it/s, Materializing param=model.layers.17.self_attn.k_proj.weight] Loading weights: 41%|████ | 127/311 [00:00<00:00, 255.40it/s, Materializing param=model.layers.11.mlp.up_proj.weight] Loading weights: 41%|████ | 127/311 [00:00<00:00, 255.40it/s, Materializing param=model.layers.11.mlp.up_proj.weight] Loading weights: 63%|██████▎ | 196/311 [00:00<00:00, 372.07it/s, Materializing param=model.layers.17.self_attn.k_proj.weight] Loading weights: 63%|██████▎ | 197/311 [00:00<00:00, 348.61it/s, Materializing param=model.layers.17.self_attn.o_proj.weight] Loading weights: 63%|██████▎ | 197/311 [00:00<00:00, 348.61it/s, Materializing param=model.layers.17.self_attn.o_proj.weight] Loading weights: 47%|████▋ | 146/311 [00:00<00:00, 240.68it/s, Materializing param=model.layers.13.input_layernorm.weight] Loading weights: 47%|████▋ | 146/311 [00:00<00:00, 240.68it/s, Materializing param=model.layers.13.input_layernorm.weight] Loading weights: 63%|██████▎ | 197/311 [00:00<00:00, 372.07it/s, Materializing param=model.layers.17.self_attn.o_proj.weight] Loading weights: 63%|██████▎ | 197/311 [00:00<00:00, 372.07it/s, Materializing param=model.layers.17.self_attn.o_proj.weight] Loading weights: 47%|████▋ | 147/311 [00:00<00:00, 240.68it/s, Materializing param=model.layers.13.mlp.down_proj.weight] Loading weights: 47%|████▋ | 147/311 [00:00<00:00, 240.68it/s, Materializing param=model.layers.13.mlp.down_proj.weight] Loading weights: 59%|█████▊ | 182/311 [00:00<00:00, 331.54it/s, Materializing param=model.layers.16.mlp.up_proj.weight] Loading weights: 59%|█████▊ | 182/311 [00:00<00:00, 331.54it/s, Materializing param=model.layers.16.mlp.up_proj.weight] Loading weights: 64%|██████▎ | 198/311 [00:00<00:00, 348.61it/s, Materializing param=model.layers.17.self_attn.q_norm.weight] Loading weights: 64%|██████▎ | 198/311 [00:00<00:00, 348.61it/s, Materializing param=model.layers.17.self_attn.q_norm.weight] Loading weights: 64%|██████▎ | 198/311 [00:00<00:00, 372.07it/s, Materializing param=model.layers.17.self_attn.q_norm.weight] Loading weights: 64%|██████▎ | 198/311 [00:00<00:00, 372.07it/s, Materializing param=model.layers.17.self_attn.q_norm.weight] Loading weights: 64%|██████▍ | 199/311 [00:00<00:00, 348.61it/s, Materializing param=model.layers.17.self_attn.q_proj.weight] Loading weights: 64%|██████▍ | 199/311 [00:00<00:00, 348.61it/s, Materializing param=model.layers.17.self_attn.q_proj.weight] Loading weights: 33%|███▎ | 104/311 [00:00<00:00, 216.05it/s, Materializing param=model.layers.9.mlp.gate_proj.weight] Loading weights: 33%|███▎ | 104/311 [00:00<00:00, 216.05it/s, Materializing param=model.layers.9.mlp.gate_proj.weight] Loading weights: 64%|██████▍ | 200/311 [00:00<00:00, 348.61it/s, Materializing param=model.layers.17.self_attn.v_proj.weight] Loading weights: 64%|██████▍ | 200/311 [00:00<00:00, 348.61it/s, Materializing param=model.layers.17.self_attn.v_proj.weight] Loading weights: 65%|██████▍ | 201/311 [00:00<00:00, 348.61it/s, Materializing param=model.layers.18.input_layernorm.weight] Loading weights: 64%|██████▍ | 199/311 [00:00<00:00, 372.07it/s, Materializing param=model.layers.17.self_attn.q_proj.weight] Loading weights: 65%|██████▍ | 201/311 [00:00<00:00, 348.61it/s, Materializing param=model.layers.18.input_layernorm.weight] Loading weights: 64%|██████▍ | 199/311 [00:00<00:00, 372.07it/s, Materializing param=model.layers.17.self_attn.q_proj.weight] Loading weights: 58%|█████▊ | 181/311 [00:00<00:00, 349.86it/s, Materializing param=model.layers.16.mlp.down_proj.weight] Loading weights: 64%|██████▍ | 200/311 [00:00<00:00, 372.07it/s, Materializing param=model.layers.17.self_attn.v_proj.weight] Loading weights: 65%|██████▍ | 202/311 [00:00<00:00, 348.61it/s, Materializing param=model.layers.18.mlp.down_proj.weight] Loading weights: 64%|██████▍ | 200/311 [00:00<00:00, 372.07it/s, Materializing param=model.layers.17.self_attn.v_proj.weight] Loading weights: 65%|██████▍ | 202/311 [00:00<00:00, 348.61it/s, Materializing param=model.layers.18.mlp.down_proj.weight] Loading weights: 58%|█████▊ | 181/311 [00:00<00:00, 349.86it/s, Materializing param=model.layers.16.mlp.gate_proj.weight] Loading weights: 58%|█████▊ | 181/311 [00:00<00:00, 349.86it/s, Materializing param=model.layers.16.mlp.gate_proj.weight] Loading weights: 65%|██████▍ | 201/311 [00:00<00:00, 372.07it/s, Materializing param=model.layers.18.input_layernorm.weight] Loading weights: 65%|██████▍ | 201/311 [00:00<00:00, 372.07it/s, Materializing param=model.layers.18.input_layernorm.weight] Loading weights: 59%|█████▊ | 182/311 [00:00<00:00, 349.86it/s, Materializing param=model.layers.16.mlp.up_proj.weight] Loading weights: 59%|█████▊ | 182/311 [00:00<00:00, 349.86it/s, Materializing param=model.layers.16.mlp.up_proj.weight] Loading weights: 65%|██████▍ | 202/311 [00:00<00:00, 372.07it/s, Materializing param=model.layers.18.mlp.down_proj.weight] Loading weights: 65%|██████▍ | 202/311 [00:00<00:00, 372.07it/s, Materializing param=model.layers.18.mlp.down_proj.weight] Loading weights: 41%|████ | 128/311 [00:00<00:00, 255.40it/s, Materializing param=model.layers.11.post_attention_layernorm.weight] Loading weights: 41%|████ | 128/311 [00:00<00:00, 255.40it/s, Materializing param=model.layers.11.post_attention_layernorm.weight] Loading weights: 41%|████▏ | 129/311 [00:00<00:00, 255.40it/s, Materializing param=model.layers.11.self_attn.k_norm.weight] Loading weights: 41%|████▏ | 129/311 [00:00<00:00, 255.40it/s, Materializing param=model.layers.11.self_attn.k_norm.weight] Loading weights: 42%|████▏ | 130/311 [00:00<00:00, 255.40it/s, Materializing param=model.layers.11.self_attn.k_proj.weight] Loading weights: 42%|████▏ | 130/311 [00:00<00:00, 255.40it/s, Materializing param=model.layers.11.self_attn.k_proj.weight] Loading weights: 42%|████▏ | 131/311 [00:00<00:00, 255.40it/s, Materializing param=model.layers.11.self_attn.o_proj.weight] Loading weights: 42%|████▏ | 131/311 [00:00<00:00, 255.40it/s, Materializing param=model.layers.11.self_attn.o_proj.weight] Loading weights: 59%|█████▉ | 183/311 [00:00<00:00, 331.54it/s, Materializing param=model.layers.16.post_attention_layernorm.weight] Loading weights: 59%|█████▉ | 183/311 [00:00<00:00, 331.54it/s, Materializing param=model.layers.16.post_attention_layernorm.weight] Loading weights: 59%|█████▉ | 184/311 [00:00<00:00, 331.54it/s, Materializing param=model.layers.16.self_attn.k_norm.weight] Loading weights: 59%|█████▉ | 184/311 [00:00<00:00, 331.54it/s, Materializing param=model.layers.16.self_attn.k_norm.weight] Loading weights: 59%|█████▉ | 185/311 [00:00<00:00, 331.54it/s, Materializing param=model.layers.16.self_attn.k_proj.weight] Loading weights: 59%|█████▉ | 185/311 [00:00<00:00, 331.54it/s, Materializing param=model.layers.16.self_attn.k_proj.weight] Loading weights: 60%|█████▉ | 186/311 [00:00<00:00, 331.54it/s, Materializing param=model.layers.16.self_attn.o_proj.weight] Loading weights: 60%|█████▉ | 186/311 [00:00<00:00, 331.54it/s, Materializing param=model.layers.16.self_attn.o_proj.weight] Loading weights: 42%|████▏ | 132/311 [00:00<00:00, 255.40it/s, Materializing param=model.layers.11.self_attn.q_norm.weight] Loading weights: 42%|████▏ | 132/311 [00:00<00:00, 255.40it/s, Materializing param=model.layers.11.self_attn.q_norm.weight] Loading weights: 65%|██████▌ | 203/311 [00:00<00:00, 372.07it/s, Materializing param=model.layers.18.mlp.gate_proj.weight] Loading weights: 59%|█████▉ | 183/311 [00:00<00:00, 349.86it/s, Materializing param=model.layers.16.post_attention_layernorm.weight] Loading weights: 34%|███▍ | 105/311 [00:00<00:00, 216.05it/s, Materializing param=model.layers.9.mlp.up_proj.weight] Loading weights: 65%|██████▌ | 203/311 [00:00<00:00, 372.07it/s, Materializing param=model.layers.18.mlp.gate_proj.weight] Loading weights: 34%|███▍ | 105/311 [00:00<00:00, 216.05it/s, Materializing param=model.layers.9.mlp.up_proj.weight] Loading weights: 59%|█████▉ | 183/311 [00:00<00:00, 349.86it/s, Materializing param=model.layers.16.post_attention_layernorm.weight] Loading weights: 60%|██████ | 187/311 [00:00<00:00, 331.54it/s, Materializing param=model.layers.16.self_attn.q_norm.weight] Loading weights: 59%|█████▉ | 184/311 [00:00<00:00, 349.86it/s, Materializing param=model.layers.16.self_attn.k_norm.weight] Loading weights: 60%|██████ | 187/311 [00:00<00:00, 331.54it/s, Materializing param=model.layers.16.self_attn.q_norm.weight] Loading weights: 59%|█████▉ | 184/311 [00:00<00:00, 349.86it/s, Materializing param=model.layers.16.self_attn.k_norm.weight] Loading weights: 60%|██████ | 188/311 [00:00<00:00, 331.54it/s, Materializing param=model.layers.16.self_attn.q_proj.weight] Loading weights: 60%|██████ | 188/311 [00:00<00:00, 331.54it/s, Materializing param=model.layers.16.self_attn.q_proj.weight] Loading weights: 59%|█████▉ | 185/311 [00:00<00:00, 349.86it/s, Materializing param=model.layers.16.self_attn.k_proj.weight] Loading weights: 59%|█████▉ | 185/311 [00:00<00:00, 349.86it/s, Materializing param=model.layers.16.self_attn.k_proj.weight] Loading weights: 43%|████▎ | 133/311 [00:00<00:00, 255.40it/s, Materializing param=model.layers.11.self_attn.q_proj.weight] Loading weights: 43%|████▎ | 133/311 [00:00<00:00, 255.40it/s, Materializing param=model.layers.11.self_attn.q_proj.weight] Loading weights: 43%|████▎ | 134/311 [00:00<00:00, 255.40it/s, Materializing param=model.layers.11.self_attn.v_proj.weight] Loading weights: 43%|████▎ | 134/311 [00:00<00:00, 255.40it/s, Materializing param=model.layers.11.self_attn.v_proj.weight] Loading weights: 61%|██████ | 189/311 [00:00<00:00, 331.54it/s, Materializing param=model.layers.16.self_attn.v_proj.weight] Loading weights: 48%|████▊ | 148/311 [00:00<00:00, 277.44it/s, Materializing param=model.layers.13.mlp.down_proj.weight] Loading weights: 61%|██████ | 189/311 [00:00<00:00, 331.54it/s, Materializing param=model.layers.16.self_attn.v_proj.weight] Loading weights: 48%|████▊ | 148/311 [00:00<00:00, 277.44it/s, Materializing param=model.layers.13.mlp.gate_proj.weight] Loading weights: 48%|████▊ | 148/311 [00:00<00:00, 277.44it/s, Materializing param=model.layers.13.mlp.gate_proj.weight] Loading weights: 48%|████▊ | 149/311 [00:00<00:00, 277.44it/s, Materializing param=model.layers.13.mlp.up_proj.weight] Loading weights: 48%|████▊ | 149/311 [00:00<00:00, 277.44it/s, Materializing param=model.layers.13.mlp.up_proj.weight] Loading weights: 43%|████▎ | 135/311 [00:00<00:00, 255.40it/s, Materializing param=model.layers.12.input_layernorm.weight] Loading weights: 43%|████▎ | 135/311 [00:00<00:00, 255.40it/s, Materializing param=model.layers.12.input_layernorm.weight] Loading weights: 60%|█████▉ | 186/311 [00:00<00:00, 349.86it/s, Materializing param=model.layers.16.self_attn.o_proj.weight] Loading weights: 44%|████▎ | 136/311 [00:00<00:00, 255.40it/s, Materializing param=model.layers.12.mlp.down_proj.weight] Loading weights: 44%|████▎ | 136/311 [00:00<00:00, 255.40it/s, Materializing param=model.layers.12.mlp.down_proj.weight] Loading weights: 60%|█████▉ | 186/311 [00:00<00:00, 349.86it/s, Materializing param=model.layers.16.self_attn.o_proj.weight] Loading weights: 61%|██████ | 190/311 [00:00<00:00, 331.54it/s, Materializing param=model.layers.17.input_layernorm.weight] Loading weights: 60%|██████ | 187/311 [00:00<00:00, 349.86it/s, Materializing param=model.layers.16.self_attn.q_norm.weight] Loading weights: 65%|██████▌ | 203/311 [00:00<00:00, 348.61it/s, Materializing param=model.layers.18.mlp.gate_proj.weight] Loading weights: 61%|██████ | 190/311 [00:00<00:00, 331.54it/s, Materializing param=model.layers.17.input_layernorm.weight] Loading weights: 65%|██████▌ | 203/311 [00:00<00:00, 348.61it/s, Materializing param=model.layers.18.mlp.gate_proj.weight] Loading weights: 60%|██████ | 187/311 [00:00<00:00, 349.86it/s, Materializing param=model.layers.16.self_attn.q_norm.weight] Loading weights: 61%|██████▏ | 191/311 [00:00<00:00, 331.54it/s, Materializing param=model.layers.17.mlp.down_proj.weight] Loading weights: 66%|██████▌ | 204/311 [00:00<00:00, 348.61it/s, Materializing param=model.layers.18.mlp.up_proj.weight] Loading weights: 60%|██████ | 188/311 [00:00<00:00, 349.86it/s, Materializing param=model.layers.16.self_attn.q_proj.weight] Loading weights: 66%|██████▌ | 204/311 [00:00<00:00, 348.61it/s, Materializing param=model.layers.18.mlp.up_proj.weight] Loading weights: 61%|██████▏ | 191/311 [00:00<00:00, 331.54it/s, Materializing param=model.layers.17.mlp.down_proj.weight] Loading weights: 66%|██████▌ | 204/311 [00:00<00:00, 372.07it/s, Materializing param=model.layers.18.mlp.up_proj.weight] Loading weights: 60%|██████ | 188/311 [00:00<00:00, 349.86it/s, Materializing param=model.layers.16.self_attn.q_proj.weight] Loading weights: 66%|██████▌ | 204/311 [00:00<00:00, 372.07it/s, Materializing param=model.layers.18.mlp.up_proj.weight] Loading weights: 61%|██████ | 189/311 [00:00<00:00, 349.86it/s, Materializing param=model.layers.16.self_attn.v_proj.weight] Loading weights: 61%|██████ | 189/311 [00:00<00:00, 349.86it/s, Materializing param=model.layers.16.self_attn.v_proj.weight] Loading weights: 61%|██████ | 190/311 [00:00<00:00, 349.86it/s, Materializing param=model.layers.17.input_layernorm.weight] Loading weights: 61%|██████ | 190/311 [00:00<00:00, 349.86it/s, Materializing param=model.layers.17.input_layernorm.weight] Loading weights: 61%|██████▏ | 191/311 [00:00<00:00, 349.86it/s, Materializing param=model.layers.17.mlp.down_proj.weight] Loading weights: 61%|██████▏ | 191/311 [00:00<00:00, 349.86it/s, Materializing param=model.layers.17.mlp.down_proj.weight] Loading weights: 34%|███▍ | 106/311 [00:00<00:00, 216.05it/s, Materializing param=model.layers.9.post_attention_layernorm.weight] Loading weights: 34%|███▍ | 106/311 [00:00<00:00, 216.05it/s, Materializing param=model.layers.9.post_attention_layernorm.weight] Loading weights: 34%|███▍ | 107/311 [00:00<00:00, 216.05it/s, Materializing param=model.layers.9.self_attn.k_norm.weight] Loading weights: 34%|███▍ | 107/311 [00:00<00:00, 216.05it/s, Materializing param=model.layers.9.self_attn.k_norm.weight] Loading weights: 35%|███▍ | 108/311 [00:00<00:00, 216.05it/s, Materializing param=model.layers.9.self_attn.k_proj.weight] Loading weights: 35%|███▍ | 108/311 [00:00<00:00, 216.05it/s, Materializing param=model.layers.9.self_attn.k_proj.weight] Loading weights: 35%|███▌ | 109/311 [00:00<00:00, 216.05it/s, Materializing param=model.layers.9.self_attn.o_proj.weight] Loading weights: 35%|███▌ | 109/311 [00:00<00:00, 216.05it/s, Materializing param=model.layers.9.self_attn.o_proj.weight] Loading weights: 48%|████▊ | 150/311 [00:00<00:00, 277.44it/s, Materializing param=model.layers.13.post_attention_layernorm.weight] Loading weights: 62%|██████▏ | 192/311 [00:00<00:00, 331.54it/s, Materializing param=model.layers.17.mlp.gate_proj.weight] Loading weights: 48%|████▊ | 150/311 [00:00<00:00, 277.44it/s, Materializing param=model.layers.13.post_attention_layernorm.weight] Loading weights: 44%|████▍ | 137/311 [00:00<00:00, 255.40it/s, Materializing param=model.layers.12.mlp.gate_proj.weight] Loading weights: 62%|██████▏ | 192/311 [00:00<00:00, 331.54it/s, Materializing param=model.layers.17.mlp.gate_proj.weight] Loading weights: 44%|████▍ | 137/311 [00:00<00:00, 255.40it/s, Materializing param=model.layers.12.mlp.gate_proj.weight] Loading weights: 49%|████▊ | 151/311 [00:00<00:00, 277.44it/s, Materializing param=model.layers.13.self_attn.k_norm.weight] Loading weights: 49%|████▊ | 151/311 [00:00<00:00, 277.44it/s, Materializing param=model.layers.13.self_attn.k_norm.weight] Loading weights: 66%|██████▌ | 205/311 [00:00<00:00, 372.07it/s, Materializing param=model.layers.18.post_attention_layernorm.weight] Loading weights: 66%|██████▌ | 205/311 [00:00<00:00, 372.07it/s, Materializing param=model.layers.18.post_attention_layernorm.weight] Loading weights: 66%|██████▌ | 205/311 [00:00<00:00, 348.61it/s, Materializing param=model.layers.18.post_attention_layernorm.weight] Loading weights: 66%|██████▌ | 205/311 [00:00<00:00, 348.61it/s, Materializing param=model.layers.18.post_attention_layernorm.weight] Loading weights: 66%|██████▌ | 206/311 [00:00<00:00, 372.07it/s, Materializing param=model.layers.18.self_attn.k_norm.weight] Loading weights: 49%|████▉ | 152/311 [00:00<00:00, 277.44it/s, Materializing param=model.layers.13.self_attn.k_proj.weight] Loading weights: 66%|██████▌ | 206/311 [00:00<00:00, 372.07it/s, Materializing param=model.layers.18.self_attn.k_norm.weight] Loading weights: 49%|████▉ | 152/311 [00:00<00:00, 277.44it/s, Materializing param=model.layers.13.self_attn.k_proj.weight] Loading weights: 66%|██████▌ | 206/311 [00:00<00:00, 348.61it/s, Materializing param=model.layers.18.self_attn.k_norm.weight] Loading weights: 66%|██████▌ | 206/311 [00:00<00:00, 348.61it/s, Materializing param=model.layers.18.self_attn.k_norm.weight] Loading weights: 67%|██████▋ | 207/311 [00:00<00:00, 372.07it/s, Materializing param=model.layers.18.self_attn.k_proj.weight] Loading weights: 35%|███▌ | 110/311 [00:00<00:00, 216.05it/s, Materializing param=model.layers.9.self_attn.q_norm.weight] Loading weights: 67%|██████▋ | 207/311 [00:00<00:00, 372.07it/s, Materializing param=model.layers.18.self_attn.k_proj.weight] Loading weights: 35%|███▌ | 110/311 [00:00<00:00, 216.05it/s, Materializing param=model.layers.9.self_attn.q_norm.weight] Loading weights: 67%|██████▋ | 207/311 [00:00<00:00, 348.61it/s, Materializing param=model.layers.18.self_attn.k_proj.weight] Loading weights: 67%|██████▋ | 207/311 [00:00<00:00, 348.61it/s, Materializing param=model.layers.18.self_attn.k_proj.weight] Loading weights: 36%|███▌ | 111/311 [00:00<00:00, 216.05it/s, Materializing param=model.layers.9.self_attn.q_proj.weight] Loading weights: 36%|███▌ | 111/311 [00:00<00:00, 216.05it/s, Materializing param=model.layers.9.self_attn.q_proj.weight] Loading weights: 49%|████▉ | 153/311 [00:00<00:00, 277.44it/s, Materializing param=model.layers.13.self_attn.o_proj.weight] Loading weights: 49%|████▉ | 153/311 [00:00<00:00, 277.44it/s, Materializing param=model.layers.13.self_attn.o_proj.weight] Loading weights: 67%|██████▋ | 208/311 [00:00<00:00, 372.07it/s, Materializing param=model.layers.18.self_attn.o_proj.weight] Loading weights: 67%|██████▋ | 208/311 [00:00<00:00, 348.61it/s, Materializing param=model.layers.18.self_attn.o_proj.weight] Loading weights: 67%|██████▋ | 208/311 [00:00<00:00, 372.07it/s, Materializing param=model.layers.18.self_attn.o_proj.weight] Loading weights: 67%|██████▋ | 208/311 [00:00<00:00, 348.61it/s, Materializing param=model.layers.18.self_attn.o_proj.weight] Loading weights: 62%|██████▏ | 192/311 [00:00<00:00, 349.86it/s, Materializing param=model.layers.17.mlp.gate_proj.weight] Loading weights: 62%|██████▏ | 192/311 [00:00<00:00, 349.86it/s, Materializing param=model.layers.17.mlp.gate_proj.weight] Loading weights: 36%|███▌ | 112/311 [00:00<00:00, 216.05it/s, Materializing param=model.layers.9.self_attn.v_proj.weight] Loading weights: 36%|███▌ | 112/311 [00:00<00:00, 216.05it/s, Materializing param=model.layers.9.self_attn.v_proj.weight] Loading weights: 50%|████▉ | 154/311 [00:00<00:00, 277.44it/s, Materializing param=model.layers.13.self_attn.q_norm.weight] Loading weights: 50%|████▉ | 154/311 [00:00<00:00, 277.44it/s, Materializing param=model.layers.13.self_attn.q_norm.weight] Loading weights: 50%|████▉ | 155/311 [00:00<00:00, 277.44it/s, Materializing param=model.layers.13.self_attn.q_proj.weight] Loading weights: 50%|████▉ | 155/311 [00:00<00:00, 277.44it/s, Materializing param=model.layers.13.self_attn.q_proj.weight] Loading weights: 67%|██████▋ | 209/311 [00:00<00:00, 348.61it/s, Materializing param=model.layers.18.self_attn.q_norm.weight] Loading weights: 67%|██████▋ | 209/311 [00:00<00:00, 348.61it/s, Materializing param=model.layers.18.self_attn.q_norm.weight] Loading weights: 67%|██████▋ | 209/311 [00:00<00:00, 372.07it/s, Materializing param=model.layers.18.self_attn.q_norm.weight] Loading weights: 67%|██████▋ | 209/311 [00:00<00:00, 372.07it/s, Materializing param=model.layers.18.self_attn.q_norm.weight] Loading weights: 36%|███▋ | 113/311 [00:00<00:00, 216.05it/s, Materializing param=model.layers.10.input_layernorm.weight] Loading weights: 36%|███▋ | 113/311 [00:00<00:00, 216.05it/s, Materializing param=model.layers.10.input_layernorm.weight] Loading weights: 37%|███▋ | 114/311 [00:00<00:00, 216.05it/s, Materializing param=model.layers.10.mlp.down_proj.weight] Loading weights: 37%|███▋ | 114/311 [00:00<00:00, 216.05it/s, Materializing param=model.layers.10.mlp.down_proj.weight] Loading weights: 50%|█████ | 156/311 [00:00<00:00, 277.44it/s, Materializing param=model.layers.13.self_attn.v_proj.weight] Loading weights: 50%|█████ | 156/311 [00:00<00:00, 277.44it/s, Materializing param=model.layers.13.self_attn.v_proj.weight] Loading weights: 68%|██████▊ | 210/311 [00:00<00:00, 348.61it/s, Materializing param=model.layers.18.self_attn.q_proj.weight] Loading weights: 68%|██████▊ | 210/311 [00:00<00:00, 348.61it/s, Materializing param=model.layers.18.self_attn.q_proj.weight] Loading weights: 68%|██████▊ | 210/311 [00:00<00:00, 372.07it/s, Materializing param=model.layers.18.self_attn.q_proj.weight] Loading weights: 68%|██████▊ | 210/311 [00:00<00:00, 372.07it/s, Materializing param=model.layers.18.self_attn.q_proj.weight] Loading weights: 62%|██████▏ | 193/311 [00:00<00:00, 331.54it/s, Materializing param=model.layers.17.mlp.up_proj.weight] Loading weights: 62%|██████▏ | 193/311 [00:00<00:00, 331.54it/s, Materializing param=model.layers.17.mlp.up_proj.weight] Loading weights: 68%|██████▊ | 211/311 [00:00<00:00, 372.07it/s, Materializing param=model.layers.18.self_attn.v_proj.weight] Loading weights: 68%|██████▊ | 211/311 [00:00<00:00, 372.07it/s, Materializing param=model.layers.18.self_attn.v_proj.weight] Loading weights: 68%|██████▊ | 211/311 [00:00<00:00, 348.61it/s, Materializing param=model.layers.18.self_attn.v_proj.weight] Loading weights: 68%|██████▊ | 211/311 [00:00<00:00, 348.61it/s, Materializing param=model.layers.18.self_attn.v_proj.weight] Loading weights: 50%|█████ | 157/311 [00:00<00:00, 277.44it/s, Materializing param=model.layers.14.input_layernorm.weight] Loading weights: 44%|████▍ | 138/311 [00:00<00:00, 255.40it/s, Materializing param=model.layers.12.mlp.up_proj.weight] Loading weights: 50%|█████ | 157/311 [00:00<00:00, 277.44it/s, Materializing param=model.layers.14.input_layernorm.weight] Loading weights: 44%|████▍ | 138/311 [00:00<00:00, 255.40it/s, Materializing param=model.layers.12.mlp.up_proj.weight] Loading weights: 68%|██████▊ | 212/311 [00:00<00:00, 348.61it/s, Materializing param=model.layers.19.input_layernorm.weight] Loading weights: 51%|█████ | 158/311 [00:00<00:00, 277.44it/s, Materializing param=model.layers.14.mlp.down_proj.weight] Loading weights: 68%|██████▊ | 212/311 [00:00<00:00, 348.61it/s, Materializing param=model.layers.19.input_layernorm.weight] Loading weights: 51%|█████ | 158/311 [00:00<00:00, 277.44it/s, Materializing param=model.layers.14.mlp.down_proj.weight] Loading weights: 68%|██████▊ | 212/311 [00:00<00:00, 372.07it/s, Materializing param=model.layers.19.input_layernorm.weight] Loading weights: 68%|██████▊ | 212/311 [00:00<00:00, 372.07it/s, Materializing param=model.layers.19.input_layernorm.weight] Loading weights: 62%|██████▏ | 193/311 [00:00<00:00, 349.86it/s, Materializing param=model.layers.17.mlp.up_proj.weight] Loading weights: 62%|██████▏ | 193/311 [00:00<00:00, 349.86it/s, Materializing param=model.layers.17.mlp.up_proj.weight] Loading weights: 37%|███▋ | 115/311 [00:00<00:00, 216.05it/s, Materializing param=model.layers.10.mlp.gate_proj.weight] Loading weights: 37%|███▋ | 115/311 [00:00<00:00, 216.05it/s, Materializing param=model.layers.10.mlp.gate_proj.weight] Loading weights: 62%|██████▏ | 194/311 [00:00<00:00, 331.54it/s, Materializing param=model.layers.17.post_attention_layernorm.weight] Loading weights: 62%|██████▏ | 194/311 [00:00<00:00, 349.86it/s, Materializing param=model.layers.17.post_attention_layernorm.weight] Loading weights: 62%|██████▏ | 194/311 [00:00<00:00, 331.54it/s, Materializing param=model.layers.17.post_attention_layernorm.weight] Loading weights: 62%|██████▏ | 194/311 [00:00<00:00, 349.86it/s, Materializing param=model.layers.17.post_attention_layernorm.weight] Loading weights: 63%|██████▎ | 195/311 [00:00<00:00, 331.54it/s, Materializing param=model.layers.17.self_attn.k_norm.weight] Loading weights: 68%|██████▊ | 213/311 [00:00<00:00, 348.61it/s, Materializing param=model.layers.19.mlp.down_proj.weight] Loading weights: 63%|██████▎ | 195/311 [00:00<00:00, 331.54it/s, Materializing param=model.layers.17.self_attn.k_norm.weight] Loading weights: 68%|██████▊ | 213/311 [00:00<00:00, 348.61it/s, Materializing param=model.layers.19.mlp.down_proj.weight] Loading weights: 68%|██████▊ | 213/311 [00:00<00:00, 372.07it/s, Materializing param=model.layers.19.mlp.down_proj.weight] Loading weights: 68%|██████▊ | 213/311 [00:00<00:00, 372.07it/s, Materializing param=model.layers.19.mlp.down_proj.weight] Loading weights: 63%|██████▎ | 195/311 [00:00<00:00, 349.86it/s, Materializing param=model.layers.17.self_attn.k_norm.weight] Loading weights: 63%|██████▎ | 196/311 [00:00<00:00, 331.54it/s, Materializing param=model.layers.17.self_attn.k_proj.weight] Loading weights: 63%|██████▎ | 196/311 [00:00<00:00, 331.54it/s, Materializing param=model.layers.17.self_attn.k_proj.weight] Loading weights: 63%|██████▎ | 195/311 [00:00<00:00, 349.86it/s, Materializing param=model.layers.17.self_attn.k_norm.weight] Loading weights: 63%|██████▎ | 196/311 [00:00<00:00, 349.86it/s, Materializing param=model.layers.17.self_attn.k_proj.weight] Loading weights: 63%|██████▎ | 196/311 [00:00<00:00, 349.86it/s, Materializing param=model.layers.17.self_attn.k_proj.weight] Loading weights: 63%|██████▎ | 197/311 [00:00<00:00, 331.54it/s, Materializing param=model.layers.17.self_attn.o_proj.weight] Loading weights: 51%|█████ | 159/311 [00:00<00:00, 277.44it/s, Materializing param=model.layers.14.mlp.gate_proj.weight] Loading weights: 63%|██████▎ | 197/311 [00:00<00:00, 331.54it/s, Materializing param=model.layers.17.self_attn.o_proj.weight] Loading weights: 51%|█████ | 159/311 [00:00<00:00, 277.44it/s, Materializing param=model.layers.14.mlp.gate_proj.weight] Loading weights: 63%|██████▎ | 197/311 [00:00<00:00, 349.86it/s, Materializing param=model.layers.17.self_attn.o_proj.weight] Loading weights: 45%|████▍ | 139/311 [00:00<00:00, 255.40it/s, Materializing param=model.layers.12.post_attention_layernorm.weight] Loading weights: 45%|████▍ | 139/311 [00:00<00:00, 255.40it/s, Materializing param=model.layers.12.post_attention_layernorm.weight] Loading weights: 63%|██████▎ | 197/311 [00:00<00:00, 349.86it/s, Materializing param=model.layers.17.self_attn.o_proj.weight] Loading weights: 45%|████▌ | 140/311 [00:00<00:00, 255.40it/s, Materializing param=model.layers.12.self_attn.k_norm.weight] Loading weights: 45%|████▌ | 140/311 [00:00<00:00, 255.40it/s, Materializing param=model.layers.12.self_attn.k_norm.weight] Loading weights: 45%|████▌ | 141/311 [00:00<00:00, 255.40it/s, Materializing param=model.layers.12.self_attn.k_proj.weight] Loading weights: 45%|████▌ | 141/311 [00:00<00:00, 255.40it/s, Materializing param=model.layers.12.self_attn.k_proj.weight] Loading weights: 46%|████▌ | 142/311 [00:00<00:00, 255.40it/s, Materializing param=model.layers.12.self_attn.o_proj.weight] Loading weights: 46%|████▌ | 142/311 [00:00<00:00, 255.40it/s, Materializing param=model.layers.12.self_attn.o_proj.weight] Loading weights: 64%|██████▎ | 198/311 [00:00<00:00, 349.86it/s, Materializing param=model.layers.17.self_attn.q_norm.weight] Loading weights: 64%|██████▎ | 198/311 [00:00<00:00, 349.86it/s, Materializing param=model.layers.17.self_attn.q_norm.weight] Loading weights: 64%|██████▍ | 199/311 [00:00<00:00, 349.86it/s, Materializing param=model.layers.17.self_attn.q_proj.weight] Loading weights: 64%|██████▍ | 199/311 [00:00<00:00, 349.86it/s, Materializing param=model.layers.17.self_attn.q_proj.weight] Loading weights: 64%|██████▎ | 198/311 [00:00<00:00, 331.54it/s, Materializing param=model.layers.17.self_attn.q_norm.weight] Loading weights: 64%|██████▎ | 198/311 [00:00<00:00, 331.54it/s, Materializing param=model.layers.17.self_attn.q_norm.weight] Loading weights: 64%|██████▍ | 199/311 [00:00<00:00, 331.54it/s, Materializing param=model.layers.17.self_attn.q_proj.weight] Loading weights: 64%|██████▍ | 199/311 [00:00<00:00, 331.54it/s, Materializing param=model.layers.17.self_attn.q_proj.weight] Loading weights: 37%|███▋ | 116/311 [00:00<00:00, 216.05it/s, Materializing param=model.layers.10.mlp.up_proj.weight] Loading weights: 37%|███▋ | 116/311 [00:00<00:00, 216.05it/s, Materializing param=model.layers.10.mlp.up_proj.weight] Loading weights: 69%|██████▉ | 214/311 [00:00<00:00, 348.61it/s, Materializing param=model.layers.19.mlp.gate_proj.weight] Loading weights: 69%|██████▉ | 214/311 [00:00<00:00, 348.61it/s, Materializing param=model.layers.19.mlp.gate_proj.weight] Loading weights: 69%|██████▉ | 214/311 [00:00<00:00, 372.07it/s, Materializing param=model.layers.19.mlp.gate_proj.weight] Loading weights: 69%|██████▉ | 214/311 [00:00<00:00, 372.07it/s, Materializing param=model.layers.19.mlp.gate_proj.weight] Loading weights: 69%|██████▉ | 215/311 [00:00<00:00, 348.61it/s, Materializing param=model.layers.19.mlp.up_proj.weight] Loading weights: 69%|██████▉ | 215/311 [00:00<00:00, 348.61it/s, Materializing param=model.layers.19.mlp.up_proj.weight] Loading weights: 69%|██████▉ | 215/311 [00:00<00:00, 372.07it/s, Materializing param=model.layers.19.mlp.up_proj.weight] Loading weights: 69%|██████▉ | 215/311 [00:00<00:00, 372.07it/s, Materializing param=model.layers.19.mlp.up_proj.weight] Loading weights: 64%|██████▍ | 200/311 [00:00<00:00, 331.54it/s, Materializing param=model.layers.17.self_attn.v_proj.weight] Loading weights: 64%|██████▍ | 200/311 [00:00<00:00, 331.54it/s, Materializing param=model.layers.17.self_attn.v_proj.weight] Loading weights: 64%|██████▍ | 200/311 [00:00<00:00, 349.86it/s, Materializing param=model.layers.17.self_attn.v_proj.weight] Loading weights: 64%|██████▍ | 200/311 [00:00<00:00, 349.86it/s, Materializing param=model.layers.17.self_attn.v_proj.weight] Loading weights: 65%|██████▍ | 201/311 [00:00<00:00, 349.86it/s, Materializing param=model.layers.18.input_layernorm.weight] Loading weights: 65%|██████▍ | 201/311 [00:00<00:00, 349.86it/s, Materializing param=model.layers.18.input_layernorm.weight] Loading weights: 65%|██████▍ | 202/311 [00:00<00:00, 349.86it/s, Materializing param=model.layers.18.mlp.down_proj.weight] Loading weights: 65%|██████▍ | 202/311 [00:00<00:00, 349.86it/s, Materializing param=model.layers.18.mlp.down_proj.weight] Loading weights: 65%|██████▍ | 201/311 [00:00<00:00, 331.54it/s, Materializing param=model.layers.18.input_layernorm.weight] Loading weights: 65%|██████▍ | 201/311 [00:00<00:00, 331.54it/s, Materializing param=model.layers.18.input_layernorm.weight] Loading weights: 65%|██████▍ | 202/311 [00:00<00:00, 331.54it/s, Materializing param=model.layers.18.mlp.down_proj.weight] Loading weights: 65%|██████▍ | 202/311 [00:00<00:00, 331.54it/s, Materializing param=model.layers.18.mlp.down_proj.weight] Loading weights: 46%|████▌ | 143/311 [00:00<00:00, 255.40it/s, Materializing param=model.layers.12.self_attn.q_norm.weight] Loading weights: 46%|████▌ | 143/311 [00:00<00:00, 255.40it/s, Materializing param=model.layers.12.self_attn.q_norm.weight] Loading weights: 46%|████▋ | 144/311 [00:00<00:00, 255.40it/s, Materializing param=model.layers.12.self_attn.q_proj.weight] Loading weights: 46%|████▋ | 144/311 [00:00<00:00, 255.40it/s, Materializing param=model.layers.12.self_attn.q_proj.weight] Loading weights: 47%|████▋ | 145/311 [00:00<00:00, 255.40it/s, Materializing param=model.layers.12.self_attn.v_proj.weight] Loading weights: 47%|████▋ | 145/311 [00:00<00:00, 255.40it/s, Materializing param=model.layers.12.self_attn.v_proj.weight] Loading weights: 51%|█████▏ | 160/311 [00:00<00:00, 277.44it/s, Materializing param=model.layers.14.mlp.up_proj.weight] Loading weights: 47%|████▋ | 146/311 [00:00<00:00, 255.40it/s, Materializing param=model.layers.13.input_layernorm.weight] Loading weights: 51%|█████▏ | 160/311 [00:00<00:00, 277.44it/s, Materializing param=model.layers.14.mlp.up_proj.weight] Loading weights: 47%|████▋ | 146/311 [00:00<00:00, 255.40it/s, Materializing param=model.layers.13.input_layernorm.weight] Loading weights: 38%|███▊ | 117/311 [00:00<00:00, 256.55it/s, Materializing param=model.layers.10.mlp.up_proj.weight] Loading weights: 38%|███▊ | 117/311 [00:00<00:00, 256.55it/s, Materializing param=model.layers.10.post_attention_layernorm.weight] Loading weights: 38%|███▊ | 117/311 [00:00<00:00, 256.55it/s, Materializing param=model.layers.10.post_attention_layernorm.weight] Loading weights: 65%|██████▌ | 203/311 [00:00<00:00, 349.86it/s, Materializing param=model.layers.18.mlp.gate_proj.weight] Loading weights: 65%|██████▌ | 203/311 [00:00<00:00, 349.86it/s, Materializing param=model.layers.18.mlp.gate_proj.weight] Loading weights: 38%|███▊ | 118/311 [00:00<00:00, 256.55it/s, Materializing param=model.layers.10.self_attn.k_norm.weight] Loading weights: 38%|███▊ | 118/311 [00:00<00:00, 256.55it/s, Materializing param=model.layers.10.self_attn.k_norm.weight] Loading weights: 69%|██████▉ | 216/311 [00:00<00:00, 372.07it/s, Materializing param=model.layers.19.post_attention_layernorm.weight] Loading weights: 38%|███▊ | 119/311 [00:00<00:00, 256.55it/s, Materializing param=model.layers.10.self_attn.k_proj.weight] Loading weights: 38%|███▊ | 119/311 [00:00<00:00, 256.55it/s, Materializing param=model.layers.10.self_attn.k_proj.weight] Loading weights: 69%|██████▉ | 216/311 [00:00<00:00, 372.07it/s, Materializing param=model.layers.19.post_attention_layernorm.weight] Loading weights: 69%|██████▉ | 216/311 [00:00<00:00, 348.61it/s, Materializing param=model.layers.19.post_attention_layernorm.weight] Loading weights: 69%|██████▉ | 216/311 [00:00<00:00, 348.61it/s, Materializing param=model.layers.19.post_attention_layernorm.weight] Loading weights: 39%|███▊ | 120/311 [00:00<00:00, 256.55it/s, Materializing param=model.layers.10.self_attn.o_proj.weight] Loading weights: 70%|██████▉ | 217/311 [00:00<00:00, 348.61it/s, Materializing param=model.layers.19.self_attn.k_norm.weight] Loading weights: 39%|███▊ | 120/311 [00:00<00:00, 256.55it/s, Materializing param=model.layers.10.self_attn.o_proj.weight] Loading weights: 70%|██████▉ | 217/311 [00:00<00:00, 348.61it/s, Materializing param=model.layers.19.self_attn.k_norm.weight] Loading weights: 70%|██████▉ | 217/311 [00:00<00:00, 372.07it/s, Materializing param=model.layers.19.self_attn.k_norm.weight] Loading weights: 70%|███████ | 218/311 [00:00<00:00, 348.61it/s, Materializing param=model.layers.19.self_attn.k_proj.weight] Loading weights: 70%|██████▉ | 217/311 [00:00<00:00, 372.07it/s, Materializing param=model.layers.19.self_attn.k_norm.weight] Loading weights: 70%|███████ | 218/311 [00:00<00:00, 348.61it/s, Materializing param=model.layers.19.self_attn.k_proj.weight] Loading weights: 70%|███████ | 218/311 [00:00<00:00, 372.07it/s, Materializing param=model.layers.19.self_attn.k_proj.weight] Loading weights: 70%|███████ | 218/311 [00:00<00:00, 372.07it/s, Materializing param=model.layers.19.self_attn.k_proj.weight] Loading weights: 70%|███████ | 219/311 [00:00<00:00, 372.07it/s, Materializing param=model.layers.19.self_attn.o_proj.weight] Loading weights: 70%|███████ | 219/311 [00:00<00:00, 348.61it/s, Materializing param=model.layers.19.self_attn.o_proj.weight] Loading weights: 70%|███████ | 219/311 [00:00<00:00, 372.07it/s, Materializing param=model.layers.19.self_attn.o_proj.weight] Loading weights: 70%|███████ | 219/311 [00:00<00:00, 348.61it/s, Materializing param=model.layers.19.self_attn.o_proj.weight] Loading weights: 65%|██████▌ | 203/311 [00:00<00:00, 331.54it/s, Materializing param=model.layers.18.mlp.gate_proj.weight] Loading weights: 65%|██████▌ | 203/311 [00:00<00:00, 331.54it/s, Materializing param=model.layers.18.mlp.gate_proj.weight] Loading weights: 52%|█████▏ | 161/311 [00:00<00:00, 277.44it/s, Materializing param=model.layers.14.post_attention_layernorm.weight] Loading weights: 52%|█████▏ | 161/311 [00:00<00:00, 277.44it/s, Materializing param=model.layers.14.post_attention_layernorm.weight] Loading weights: 52%|█████▏ | 162/311 [00:00<00:00, 277.44it/s, Materializing param=model.layers.14.self_attn.k_norm.weight] Loading weights: 39%|███▉ | 121/311 [00:00<00:00, 256.55it/s, Materializing param=model.layers.10.self_attn.q_norm.weight] Loading weights: 52%|█████▏ | 162/311 [00:00<00:00, 277.44it/s, Materializing param=model.layers.14.self_attn.k_norm.weight] Loading weights: 39%|███▉ | 121/311 [00:00<00:00, 256.55it/s, Materializing param=model.layers.10.self_attn.q_norm.weight] Loading weights: 52%|█████▏ | 163/311 [00:00<00:00, 277.44it/s, Materializing param=model.layers.14.self_attn.k_proj.weight] Loading weights: 39%|███▉ | 122/311 [00:00<00:00, 256.55it/s, Materializing param=model.layers.10.self_attn.q_proj.weight] Loading weights: 52%|█████▏ | 163/311 [00:00<00:00, 277.44it/s, Materializing param=model.layers.14.self_attn.k_proj.weight] Loading weights: 39%|███▉ | 122/311 [00:00<00:00, 256.55it/s, Materializing param=model.layers.10.self_attn.q_proj.weight] Loading weights: 71%|███████ | 220/311 [00:00<00:00, 372.07it/s, Materializing param=model.layers.19.self_attn.q_norm.weight] Loading weights: 71%|███████ | 220/311 [00:00<00:00, 372.07it/s, Materializing param=model.layers.19.self_attn.q_norm.weight] Loading weights: 71%|███████ | 221/311 [00:00<00:00, 372.07it/s, Materializing param=model.layers.19.self_attn.q_proj.weight] Loading weights: 53%|█████▎ | 164/311 [00:00<00:00, 277.44it/s, Materializing param=model.layers.14.self_attn.o_proj.weight] Loading weights: 71%|███████ | 221/311 [00:00<00:00, 372.07it/s, Materializing param=model.layers.19.self_attn.q_proj.weight] Loading weights: 53%|█████▎ | 164/311 [00:00<00:00, 277.44it/s, Materializing param=model.layers.14.self_attn.o_proj.weight] Loading weights: 71%|███████ | 220/311 [00:00<00:00, 348.61it/s, Materializing param=model.layers.19.self_attn.q_norm.weight] Loading weights: 71%|███████ | 220/311 [00:00<00:00, 348.61it/s, Materializing param=model.layers.19.self_attn.q_norm.weight] Loading weights: 71%|███████ | 221/311 [00:00<00:00, 348.61it/s, Materializing param=model.layers.19.self_attn.q_proj.weight] Loading weights: 71%|███████ | 221/311 [00:00<00:00, 348.61it/s, Materializing param=model.layers.19.self_attn.q_proj.weight] Loading weights: 71%|███████▏ | 222/311 [00:00<00:00, 348.61it/s, Materializing param=model.layers.19.self_attn.v_proj.weight] Loading weights: 71%|███████▏ | 222/311 [00:00<00:00, 348.61it/s, Materializing param=model.layers.19.self_attn.v_proj.weight] Loading weights: 71%|███████▏ | 222/311 [00:00<00:00, 372.07it/s, Materializing param=model.layers.19.self_attn.v_proj.weight] Loading weights: 71%|███████▏ | 222/311 [00:00<00:00, 372.07it/s, Materializing param=model.layers.19.self_attn.v_proj.weight] Loading weights: 40%|███▉ | 123/311 [00:00<00:00, 256.55it/s, Materializing param=model.layers.10.self_attn.v_proj.weight] Loading weights: 40%|███▉ | 123/311 [00:00<00:00, 256.55it/s, Materializing param=model.layers.10.self_attn.v_proj.weight] Loading weights: 72%|███████▏ | 223/311 [00:00<00:00, 348.61it/s, Materializing param=model.layers.20.input_layernorm.weight] Loading weights: 72%|███████▏ | 223/311 [00:00<00:00, 348.61it/s, Materializing param=model.layers.20.input_layernorm.weight] Loading weights: 72%|███████▏ | 224/311 [00:00<00:00, 348.61it/s, Materializing param=model.layers.20.mlp.down_proj.weight] Loading weights: 66%|██████▌ | 204/311 [00:00<00:00, 349.86it/s, Materializing param=model.layers.18.mlp.up_proj.weight] Loading weights: 72%|███████▏ | 224/311 [00:00<00:00, 348.61it/s, Materializing param=model.layers.20.mlp.down_proj.weight] Loading weights: 66%|██████▌ | 204/311 [00:00<00:00, 349.86it/s, Materializing param=model.layers.18.mlp.up_proj.weight] Loading weights: 53%|█████▎ | 165/311 [00:00<00:00, 277.44it/s, Materializing param=model.layers.14.self_attn.q_norm.weight] Loading weights: 72%|███████▏ | 223/311 [00:00<00:00, 372.07it/s, Materializing param=model.layers.20.input_layernorm.weight] Loading weights: 53%|█████▎ | 165/311 [00:00<00:00, 277.44it/s, Materializing param=model.layers.14.self_attn.q_norm.weight] Loading weights: 72%|███████▏ | 223/311 [00:00<00:00, 372.07it/s, Materializing param=model.layers.20.input_layernorm.weight] Loading weights: 72%|███████▏ | 224/311 [00:00<00:00, 372.07it/s, Materializing param=model.layers.20.mlp.down_proj.weight] Loading weights: 72%|███████▏ | 224/311 [00:00<00:00, 372.07it/s, Materializing param=model.layers.20.mlp.down_proj.weight] Loading weights: 40%|███▉ | 124/311 [00:00<00:00, 256.55it/s, Materializing param=model.layers.11.input_layernorm.weight] Loading weights: 40%|███▉ | 124/311 [00:00<00:00, 256.55it/s, Materializing param=model.layers.11.input_layernorm.weight] Loading weights: 53%|█████▎ | 166/311 [00:00<00:00, 277.44it/s, Materializing param=model.layers.14.self_attn.q_proj.weight] Loading weights: 53%|█████▎ | 166/311 [00:00<00:00, 277.44it/s, Materializing param=model.layers.14.self_attn.q_proj.weight] Loading weights: 54%|█████▎ | 167/311 [00:00<00:00, 277.44it/s, Materializing param=model.layers.14.self_attn.v_proj.weight] Loading weights: 54%|█████▎ | 167/311 [00:00<00:00, 277.44it/s, Materializing param=model.layers.14.self_attn.v_proj.weight] Loading weights: 66%|██████▌ | 204/311 [00:00<00:00, 331.54it/s, Materializing param=model.layers.18.mlp.up_proj.weight] Loading weights: 66%|██████▌ | 204/311 [00:00<00:00, 331.54it/s, Materializing param=model.layers.18.mlp.up_proj.weight] Loading weights: 54%|█████▍ | 168/311 [00:00<00:00, 277.44it/s, Materializing param=model.layers.15.input_layernorm.weight] Loading weights: 54%|█████▍ | 168/311 [00:00<00:00, 277.44it/s, Materializing param=model.layers.15.input_layernorm.weight] Loading weights: 54%|█████▍ | 169/311 [00:00<00:00, 277.44it/s, Materializing param=model.layers.15.mlp.down_proj.weight] Loading weights: 54%|█████▍ | 169/311 [00:00<00:00, 277.44it/s, Materializing param=model.layers.15.mlp.down_proj.weight] Loading weights: 47%|████▋ | 147/311 [00:00<00:00, 255.40it/s, Materializing param=model.layers.13.mlp.down_proj.weight] Loading weights: 47%|████▋ | 147/311 [00:00<00:00, 255.40it/s, Materializing param=model.layers.13.mlp.down_proj.weight] Loading weights: 48%|████▊ | 148/311 [00:00<00:00, 255.40it/s, Materializing param=model.layers.13.mlp.gate_proj.weight] Loading weights: 48%|████▊ | 148/311 [00:00<00:00, 255.40it/s, Materializing param=model.layers.13.mlp.gate_proj.weight] Loading weights: 66%|██████▌ | 205/311 [00:00<00:00, 349.86it/s, Materializing param=model.layers.18.post_attention_layernorm.weight] Loading weights: 48%|████▊ | 149/311 [00:00<00:00, 255.40it/s, Materializing param=model.layers.13.mlp.up_proj.weight] Loading weights: 48%|████▊ | 149/311 [00:00<00:00, 255.40it/s, Materializing param=model.layers.13.mlp.up_proj.weight] Loading weights: 66%|██████▌ | 205/311 [00:00<00:00, 349.86it/s, Materializing param=model.layers.18.post_attention_layernorm.weight] Loading weights: 72%|███████▏ | 225/311 [00:00<00:00, 372.07it/s, Materializing param=model.layers.20.mlp.gate_proj.weight] Loading weights: 72%|███████▏ | 225/311 [00:00<00:00, 348.61it/s, Materializing param=model.layers.20.mlp.gate_proj.weight] Loading weights: 72%|███████▏ | 225/311 [00:00<00:00, 372.07it/s, Materializing param=model.layers.20.mlp.gate_proj.weight] Loading weights: 66%|██████▌ | 206/311 [00:00<00:00, 349.86it/s, Materializing param=model.layers.18.self_attn.k_norm.weight] Loading weights: 72%|███████▏ | 225/311 [00:00<00:00, 348.61it/s, Materializing param=model.layers.20.mlp.gate_proj.weight] Loading weights: 66%|██████▌ | 206/311 [00:00<00:00, 349.86it/s, Materializing param=model.layers.18.self_attn.k_norm.weight] Loading weights: 67%|██████▋ | 207/311 [00:00<00:00, 349.86it/s, Materializing param=model.layers.18.self_attn.k_proj.weight] Loading weights: 67%|██████▋ | 207/311 [00:00<00:00, 349.86it/s, Materializing param=model.layers.18.self_attn.k_proj.weight] Loading weights: 67%|██████▋ | 208/311 [00:00<00:00, 349.86it/s, Materializing param=model.layers.18.self_attn.o_proj.weight] Loading weights: 67%|██████▋ | 208/311 [00:00<00:00, 349.86it/s, Materializing param=model.layers.18.self_attn.o_proj.weight] Loading weights: 40%|████ | 125/311 [00:00<00:00, 256.55it/s, Materializing param=model.layers.11.mlp.down_proj.weight] Loading weights: 40%|████ | 125/311 [00:00<00:00, 256.55it/s, Materializing param=model.layers.11.mlp.down_proj.weight] Loading weights: 41%|████ | 126/311 [00:00<00:00, 256.55it/s, Materializing param=model.layers.11.mlp.gate_proj.weight] Loading weights: 41%|████ | 126/311 [00:00<00:00, 256.55it/s, Materializing param=model.layers.11.mlp.gate_proj.weight] Loading weights: 67%|██████▋ | 209/311 [00:00<00:00, 349.86it/s, Materializing param=model.layers.18.self_attn.q_norm.weight] Loading weights: 67%|██████▋ | 209/311 [00:00<00:00, 349.86it/s, Materializing param=model.layers.18.self_attn.q_norm.weight] Loading weights: 73%|███████▎ | 226/311 [00:00<00:00, 348.61it/s, Materializing param=model.layers.20.mlp.up_proj.weight] Loading weights: 73%|███████▎ | 226/311 [00:00<00:00, 348.61it/s, Materializing param=model.layers.20.mlp.up_proj.weight] Loading weights: 68%|██████▊ | 210/311 [00:00<00:00, 349.86it/s, Materializing param=model.layers.18.self_attn.q_proj.weight] Loading weights: 66%|██████▌ | 205/311 [00:00<00:00, 331.54it/s, Materializing param=model.layers.18.post_attention_layernorm.weight] Loading weights: 68%|██████▊ | 210/311 [00:00<00:00, 349.86it/s, Materializing param=model.layers.18.self_attn.q_proj.weight] Loading weights: 66%|██████▌ | 205/311 [00:00<00:00, 331.54it/s, Materializing param=model.layers.18.post_attention_layernorm.weight] Loading weights: 68%|██████▊ | 211/311 [00:00<00:00, 349.86it/s, Materializing param=model.layers.18.self_attn.v_proj.weight] Loading weights: 68%|██████▊ | 211/311 [00:00<00:00, 349.86it/s, Materializing param=model.layers.18.self_attn.v_proj.weight] Loading weights: 66%|██████▌ | 206/311 [00:00<00:00, 331.54it/s, Materializing param=model.layers.18.self_attn.k_norm.weight] Loading weights: 66%|██████▌ | 206/311 [00:00<00:00, 331.54it/s, Materializing param=model.layers.18.self_attn.k_norm.weight] Loading weights: 67%|██████▋ | 207/311 [00:00<00:00, 331.54it/s, Materializing param=model.layers.18.self_attn.k_proj.weight] Loading weights: 68%|██████▊ | 212/311 [00:00<00:00, 349.86it/s, Materializing param=model.layers.19.input_layernorm.weight] Loading weights: 67%|██████▋ | 207/311 [00:00<00:00, 331.54it/s, Materializing param=model.layers.18.self_attn.k_proj.weight] Loading weights: 68%|██████▊ | 212/311 [00:00<00:00, 349.86it/s, Materializing param=model.layers.19.input_layernorm.weight] Loading weights: 67%|██████▋ | 208/311 [00:00<00:00, 331.54it/s, Materializing param=model.layers.18.self_attn.o_proj.weight] Loading weights: 48%|████▊ | 150/311 [00:00<00:00, 255.40it/s, Materializing param=model.layers.13.post_attention_layernorm.weight] Loading weights: 67%|██████▋ | 208/311 [00:00<00:00, 331.54it/s, Materializing param=model.layers.18.self_attn.o_proj.weight] Loading weights: 48%|████▊ | 150/311 [00:00<00:00, 255.40it/s, Materializing param=model.layers.13.post_attention_layernorm.weight] Loading weights: 49%|████▊ | 151/311 [00:00<00:00, 255.40it/s, Materializing param=model.layers.13.self_attn.k_norm.weight] Loading weights: 49%|████▊ | 151/311 [00:00<00:00, 255.40it/s, Materializing param=model.layers.13.self_attn.k_norm.weight] Loading weights: 73%|███████▎ | 226/311 [00:00<00:00, 372.07it/s, Materializing param=model.layers.20.mlp.up_proj.weight] Loading weights: 73%|███████▎ | 226/311 [00:00<00:00, 372.07it/s, Materializing param=model.layers.20.mlp.up_proj.weight] Loading weights: 49%|████▉ | 152/311 [00:00<00:00, 255.40it/s, Materializing param=model.layers.13.self_attn.k_proj.weight] Loading weights: 49%|████▉ | 152/311 [00:00<00:00, 255.40it/s, Materializing param=model.layers.13.self_attn.k_proj.weight] Loading weights: 49%|████▉ | 153/311 [00:00<00:00, 255.40it/s, Materializing param=model.layers.13.self_attn.o_proj.weight] Loading weights: 49%|████▉ | 153/311 [00:00<00:00, 255.40it/s, Materializing param=model.layers.13.self_attn.o_proj.weight] Loading weights: 41%|████ | 127/311 [00:00<00:00, 256.55it/s, Materializing param=model.layers.11.mlp.up_proj.weight] Loading weights: 41%|████ | 127/311 [00:00<00:00, 256.55it/s, Materializing param=model.layers.11.mlp.up_proj.weight] Loading weights: 67%|██████▋ | 209/311 [00:00<00:00, 331.54it/s, Materializing param=model.layers.18.self_attn.q_norm.weight] Loading weights: 67%|██████▋ | 209/311 [00:00<00:00, 331.54it/s, Materializing param=model.layers.18.self_attn.q_norm.weight] Loading weights: 50%|████▉ | 154/311 [00:00<00:00, 292.69it/s, Materializing param=model.layers.13.self_attn.o_proj.weight] Loading weights: 50%|████▉ | 154/311 [00:00<00:00, 292.69it/s, Materializing param=model.layers.13.self_attn.q_norm.weight] Loading weights: 50%|████▉ | 154/311 [00:00<00:00, 292.69it/s, Materializing param=model.layers.13.self_attn.q_norm.weight] Loading weights: 50%|████▉ | 155/311 [00:00<00:00, 292.69it/s, Materializing param=model.layers.13.self_attn.q_proj.weight] Loading weights: 50%|████▉ | 155/311 [00:00<00:00, 292.69it/s, Materializing param=model.layers.13.self_attn.q_proj.weight] Loading weights: 68%|██████▊ | 210/311 [00:00<00:00, 331.54it/s, Materializing param=model.layers.18.self_attn.q_proj.weight] Loading weights: 68%|██████▊ | 210/311 [00:00<00:00, 331.54it/s, Materializing param=model.layers.18.self_attn.q_proj.weight] Loading weights: 68%|██████▊ | 211/311 [00:00<00:00, 331.54it/s, Materializing param=model.layers.18.self_attn.v_proj.weight] Loading weights: 68%|██████▊ | 211/311 [00:00<00:00, 331.54it/s, Materializing param=model.layers.18.self_attn.v_proj.weight] Loading weights: 73%|███████▎ | 227/311 [00:00<00:00, 372.07it/s, Materializing param=model.layers.20.post_attention_layernorm.weight] Loading weights: 73%|███████▎ | 227/311 [00:00<00:00, 372.07it/s, Materializing param=model.layers.20.post_attention_layernorm.weight] Loading weights: 73%|███████▎ | 227/311 [00:00<00:00, 375.64it/s, Materializing param=model.layers.20.mlp.up_proj.weight] Loading weights: 73%|███████▎ | 227/311 [00:00<00:00, 375.64it/s, Materializing param=model.layers.20.post_attention_layernorm.weight] Loading weights: 73%|███████▎ | 228/311 [00:00<00:00, 372.07it/s, Materializing param=model.layers.20.self_attn.k_norm.weight] Loading weights: 73%|███████▎ | 227/311 [00:00<00:00, 375.64it/s, Materializing param=model.layers.20.post_attention_layernorm.weight] Loading weights: 73%|███████▎ | 228/311 [00:00<00:00, 372.07it/s, Materializing param=model.layers.20.self_attn.k_norm.weight] Loading weights: 73%|███████▎ | 228/311 [00:00<00:00, 375.64it/s, Materializing param=model.layers.20.self_attn.k_norm.weight] Loading weights: 73%|███████▎ | 228/311 [00:00<00:00, 375.64it/s, Materializing param=model.layers.20.self_attn.k_norm.weight] Loading weights: 74%|███████▎ | 229/311 [00:00<00:00, 375.64it/s, Materializing param=model.layers.20.self_attn.k_proj.weight] Loading weights: 74%|███████▎ | 229/311 [00:00<00:00, 375.64it/s, Materializing param=model.layers.20.self_attn.k_proj.weight] Loading weights: 74%|███████▎ | 229/311 [00:00<00:00, 372.07it/s, Materializing param=model.layers.20.self_attn.k_proj.weight] Loading weights: 74%|███████▎ | 229/311 [00:00<00:00, 372.07it/s, Materializing param=model.layers.20.self_attn.k_proj.weight] Loading weights: 68%|██████▊ | 212/311 [00:00<00:00, 331.54it/s, Materializing param=model.layers.19.input_layernorm.weight] Loading weights: 68%|██████▊ | 212/311 [00:00<00:00, 331.54it/s, Materializing param=model.layers.19.input_layernorm.weight] Loading weights: 68%|██████▊ | 213/311 [00:00<00:00, 331.54it/s, Materializing param=model.layers.19.mlp.down_proj.weight] Loading weights: 74%|███████▍ | 230/311 [00:00<00:00, 372.07it/s, Materializing param=model.layers.20.self_attn.o_proj.weight] Loading weights: 68%|██████▊ | 213/311 [00:00<00:00, 331.54it/s, Materializing param=model.layers.19.mlp.down_proj.weight] Loading weights: 74%|███████▍ | 230/311 [00:00<00:00, 375.64it/s, Materializing param=model.layers.20.self_attn.o_proj.weight] Loading weights: 74%|███████▍ | 230/311 [00:00<00:00, 372.07it/s, Materializing param=model.layers.20.self_attn.o_proj.weight] Loading weights: 74%|███████▍ | 230/311 [00:00<00:00, 375.64it/s, Materializing param=model.layers.20.self_attn.o_proj.weight] Loading weights: 68%|██████▊ | 213/311 [00:00<00:00, 349.86it/s, Materializing param=model.layers.19.mlp.down_proj.weight] Loading weights: 68%|██████▊ | 213/311 [00:00<00:00, 349.86it/s, Materializing param=model.layers.19.mlp.down_proj.weight] Loading weights: 69%|██████▉ | 214/311 [00:00<00:00, 349.86it/s, Materializing param=model.layers.19.mlp.gate_proj.weight] Loading weights: 69%|██████▉ | 214/311 [00:00<00:00, 349.86it/s, Materializing param=model.layers.19.mlp.gate_proj.weight] Loading weights: 55%|█████▍ | 170/311 [00:00<00:00, 277.44it/s, Materializing param=model.layers.15.mlp.gate_proj.weight] Loading weights: 55%|█████▍ | 170/311 [00:00<00:00, 277.44it/s, Materializing param=model.layers.15.mlp.gate_proj.weight] Loading weights: 50%|█████ | 156/311 [00:00<00:00, 292.69it/s, Materializing param=model.layers.13.self_attn.v_proj.weight] Loading weights: 50%|█████ | 156/311 [00:00<00:00, 292.69it/s, Materializing param=model.layers.13.self_attn.v_proj.weight] Loading weights: 55%|█████▍ | 171/311 [00:00<00:00, 277.44it/s, Materializing param=model.layers.15.mlp.up_proj.weight] Loading weights: 55%|█████▍ | 171/311 [00:00<00:00, 277.44it/s, Materializing param=model.layers.15.mlp.up_proj.weight] Loading weights: 50%|█████ | 157/311 [00:00<00:00, 292.69it/s, Materializing param=model.layers.14.input_layernorm.weight] Loading weights: 50%|█████ | 157/311 [00:00<00:00, 292.69it/s, Materializing param=model.layers.14.input_layernorm.weight] Loading weights: 41%|████ | 128/311 [00:00<00:00, 256.55it/s, Materializing param=model.layers.11.post_attention_layernorm.weight] Loading weights: 41%|████ | 128/311 [00:00<00:00, 256.55it/s, Materializing param=model.layers.11.post_attention_layernorm.weight] Loading weights: 41%|████▏ | 129/311 [00:00<00:00, 256.55it/s, Materializing param=model.layers.11.self_attn.k_norm.weight] Loading weights: 41%|████▏ | 129/311 [00:00<00:00, 256.55it/s, Materializing param=model.layers.11.self_attn.k_norm.weight] Loading weights: 42%|████▏ | 130/311 [00:00<00:00, 256.55it/s, Materializing param=model.layers.11.self_attn.k_proj.weight] Loading weights: 42%|████▏ | 130/311 [00:00<00:00, 256.55it/s, Materializing param=model.layers.11.self_attn.k_proj.weight] Loading weights: 42%|████▏ | 131/311 [00:00<00:00, 256.55it/s, Materializing param=model.layers.11.self_attn.o_proj.weight] Loading weights: 42%|████▏ | 131/311 [00:00<00:00, 256.55it/s, Materializing param=model.layers.11.self_attn.o_proj.weight] Loading weights: 74%|███████▍ | 231/311 [00:00<00:00, 372.07it/s, Materializing param=model.layers.20.self_attn.q_norm.weight] Loading weights: 74%|███████▍ | 231/311 [00:00<00:00, 372.07it/s, Materializing param=model.layers.20.self_attn.q_norm.weight] Loading weights: 75%|███████▍ | 232/311 [00:00<00:00, 372.07it/s, Materializing param=model.layers.20.self_attn.q_proj.weight] Loading weights: 75%|███████▍ | 232/311 [00:00<00:00, 372.07it/s, Materializing param=model.layers.20.self_attn.q_proj.weight] Loading weights: 74%|███████▍ | 231/311 [00:00<00:00, 375.64it/s, Materializing param=model.layers.20.self_attn.q_norm.weight] Loading weights: 74%|███████▍ | 231/311 [00:00<00:00, 375.64it/s, Materializing param=model.layers.20.self_attn.q_norm.weight] Loading weights: 75%|███████▍ | 233/311 [00:00<00:00, 372.07it/s, Materializing param=model.layers.20.self_attn.v_proj.weight] Loading weights: 75%|███████▍ | 233/311 [00:00<00:00, 372.07it/s, Materializing param=model.layers.20.self_attn.v_proj.weight] Loading weights: 75%|███████▍ | 232/311 [00:00<00:00, 375.64it/s, Materializing param=model.layers.20.self_attn.q_proj.weight] Loading weights: 75%|███████▍ | 232/311 [00:00<00:00, 375.64it/s, Materializing param=model.layers.20.self_attn.q_proj.weight] Loading weights: 75%|███████▍ | 233/311 [00:00<00:00, 375.64it/s, Materializing param=model.layers.20.self_attn.v_proj.weight] Loading weights: 75%|███████▍ | 233/311 [00:00<00:00, 375.64it/s, Materializing param=model.layers.20.self_attn.v_proj.weight] Loading weights: 75%|███████▌ | 234/311 [00:00<00:00, 372.07it/s, Materializing param=model.layers.21.input_layernorm.weight] Loading weights: 75%|███████▌ | 234/311 [00:00<00:00, 372.07it/s, Materializing param=model.layers.21.input_layernorm.weight] Loading weights: 75%|███████▌ | 234/311 [00:00<00:00, 375.64it/s, Materializing param=model.layers.21.input_layernorm.weight] Loading weights: 75%|███████▌ | 234/311 [00:00<00:00, 375.64it/s, Materializing param=model.layers.21.input_layernorm.weight] Loading weights: 76%|███████▌ | 235/311 [00:00<00:00, 372.07it/s, Materializing param=model.layers.21.mlp.down_proj.weight] Loading weights: 69%|██████▉ | 215/311 [00:00<00:00, 349.86it/s, Materializing param=model.layers.19.mlp.up_proj.weight] Loading weights: 76%|███████▌ | 235/311 [00:00<00:00, 372.07it/s, Materializing param=model.layers.21.mlp.down_proj.weight] Loading weights: 69%|██████▉ | 214/311 [00:00<00:00, 331.54it/s, Materializing param=model.layers.19.mlp.gate_proj.weight] Loading weights: 69%|██████▉ | 215/311 [00:00<00:00, 349.86it/s, Materializing param=model.layers.19.mlp.up_proj.weight] Loading weights: 69%|██████▉ | 214/311 [00:00<00:00, 331.54it/s, Materializing param=model.layers.19.mlp.gate_proj.weight] Loading weights: 51%|█████ | 158/311 [00:00<00:00, 292.69it/s, Materializing param=model.layers.14.mlp.down_proj.weight] Loading weights: 51%|█████ | 158/311 [00:00<00:00, 292.69it/s, Materializing param=model.layers.14.mlp.down_proj.weight] Loading weights: 55%|█████▌ | 172/311 [00:00<00:00, 277.44it/s, Materializing param=model.layers.15.post_attention_layernorm.weight] Loading weights: 55%|█████▌ | 172/311 [00:00<00:00, 277.44it/s, Materializing param=model.layers.15.post_attention_layernorm.weight] Loading weights: 51%|█████ | 159/311 [00:00<00:00, 292.69it/s, Materializing param=model.layers.14.mlp.gate_proj.weight] Loading weights: 51%|█████ | 159/311 [00:00<00:00, 292.69it/s, Materializing param=model.layers.14.mlp.gate_proj.weight] Loading weights: 56%|█████▌ | 173/311 [00:00<00:00, 277.44it/s, Materializing param=model.layers.15.self_attn.k_norm.weight] Loading weights: 56%|█████▌ | 173/311 [00:00<00:00, 277.44it/s, Materializing param=model.layers.15.self_attn.k_norm.weight] Loading weights: 56%|█████▌ | 174/311 [00:00<00:00, 277.44it/s, Materializing param=model.layers.15.self_attn.k_proj.weight] Loading weights: 56%|█████▌ | 174/311 [00:00<00:00, 277.44it/s, Materializing param=model.layers.15.self_attn.k_proj.weight] Loading weights: 42%|████▏ | 132/311 [00:00<00:00, 256.55it/s, Materializing param=model.layers.11.self_attn.q_norm.weight] Loading weights: 42%|████▏ | 132/311 [00:00<00:00, 256.55it/s, Materializing param=model.layers.11.self_attn.q_norm.weight] Loading weights: 43%|████▎ | 133/311 [00:00<00:00, 256.55it/s, Materializing param=model.layers.11.self_attn.q_proj.weight] Loading weights: 43%|████▎ | 133/311 [00:00<00:00, 256.55it/s, Materializing param=model.layers.11.self_attn.q_proj.weight] Loading weights: 43%|████▎ | 134/311 [00:00<00:00, 256.55it/s, Materializing param=model.layers.11.self_attn.v_proj.weight] Loading weights: 43%|████▎ | 134/311 [00:00<00:00, 256.55it/s, Materializing param=model.layers.11.self_attn.v_proj.weight] Loading weights: 56%|█████▋ | 175/311 [00:00<00:00, 277.44it/s, Materializing param=model.layers.15.self_attn.o_proj.weight] Loading weights: 56%|█████▋ | 175/311 [00:00<00:00, 277.44it/s, Materializing param=model.layers.15.self_attn.o_proj.weight] Loading weights: 43%|████▎ | 135/311 [00:00<00:00, 256.55it/s, Materializing param=model.layers.12.input_layernorm.weight] Loading weights: 43%|████▎ | 135/311 [00:00<00:00, 256.55it/s, Materializing param=model.layers.12.input_layernorm.weight] Loading weights: 44%|████▎ | 136/311 [00:00<00:00, 256.55it/s, Materializing param=model.layers.12.mlp.down_proj.weight] Loading weights: 44%|████▎ | 136/311 [00:00<00:00, 256.55it/s, Materializing param=model.layers.12.mlp.down_proj.weight] Loading weights: 69%|██████▉ | 216/311 [00:00<00:00, 349.86it/s, Materializing param=model.layers.19.post_attention_layernorm.weight] Loading weights: 76%|███████▌ | 236/311 [00:00<00:00, 372.07it/s, Materializing param=model.layers.21.mlp.gate_proj.weight] Loading weights: 76%|███████▌ | 235/311 [00:00<00:00, 375.64it/s, Materializing param=model.layers.21.mlp.down_proj.weight] Loading weights: 76%|███████▌ | 236/311 [00:00<00:00, 372.07it/s, Materializing param=model.layers.21.mlp.gate_proj.weight] Loading weights: 76%|███████▌ | 235/311 [00:00<00:00, 375.64it/s, Materializing param=model.layers.21.mlp.down_proj.weight] Loading weights: 69%|██████▉ | 216/311 [00:00<00:00, 349.86it/s, Materializing param=model.layers.19.post_attention_layernorm.weight] Loading weights: 76%|███████▌ | 236/311 [00:00<00:00, 375.64it/s, Materializing param=model.layers.21.mlp.gate_proj.weight] Loading weights: 76%|███████▌ | 236/311 [00:00<00:00, 375.64it/s, Materializing param=model.layers.21.mlp.gate_proj.weight] Loading weights: 70%|██████▉ | 217/311 [00:00<00:00, 349.86it/s, Materializing param=model.layers.19.self_attn.k_norm.weight] Loading weights: 70%|██████▉ | 217/311 [00:00<00:00, 349.86it/s, Materializing param=model.layers.19.self_attn.k_norm.weight] Loading weights: 57%|█████▋ | 176/311 [00:00<00:00, 277.44it/s, Materializing param=model.layers.15.self_attn.q_norm.weight] Loading weights: 57%|█████▋ | 176/311 [00:00<00:00, 277.44it/s, Materializing param=model.layers.15.self_attn.q_norm.weight] Loading weights: 57%|█████▋ | 177/311 [00:00<00:00, 277.44it/s, Materializing param=model.layers.15.self_attn.q_proj.weight] Loading weights: 57%|█████▋ | 177/311 [00:00<00:00, 277.44it/s, Materializing param=model.layers.15.self_attn.q_proj.weight] Loading weights: 70%|███████ | 218/311 [00:00<00:00, 349.86it/s, Materializing param=model.layers.19.self_attn.k_proj.weight] Loading weights: 69%|██████▉ | 215/311 [00:00<00:00, 357.93it/s, Materializing param=model.layers.19.mlp.gate_proj.weight] Loading weights: 70%|███████ | 218/311 [00:00<00:00, 349.86it/s, Materializing param=model.layers.19.self_attn.k_proj.weight] Loading weights: 69%|██████▉ | 215/311 [00:00<00:00, 357.93it/s, Materializing param=model.layers.19.mlp.up_proj.weight] Loading weights: 69%|██████▉ | 215/311 [00:00<00:00, 357.93it/s, Materializing param=model.layers.19.mlp.up_proj.weight] Loading weights: 70%|███████ | 219/311 [00:00<00:00, 349.86it/s, Materializing param=model.layers.19.self_attn.o_proj.weight] Loading weights: 70%|███████ | 219/311 [00:00<00:00, 349.86it/s, Materializing param=model.layers.19.self_attn.o_proj.weight] Loading weights: 57%|█████▋ | 178/311 [00:00<00:00, 277.44it/s, Materializing param=model.layers.15.self_attn.v_proj.weight] Loading weights: 57%|█████▋ | 178/311 [00:00<00:00, 277.44it/s, Materializing param=model.layers.15.self_attn.v_proj.weight] Loading weights: 71%|███████ | 220/311 [00:00<00:00, 349.86it/s, Materializing param=model.layers.19.self_attn.q_norm.weight] Loading weights: 71%|███████ | 220/311 [00:00<00:00, 349.86it/s, Materializing param=model.layers.19.self_attn.q_norm.weight] Loading weights: 71%|███████ | 221/311 [00:00<00:00, 349.86it/s, Materializing param=model.layers.19.self_attn.q_proj.weight] Loading weights: 71%|███████ | 221/311 [00:00<00:00, 349.86it/s, Materializing param=model.layers.19.self_attn.q_proj.weight] Loading weights: 51%|█████▏ | 160/311 [00:00<00:00, 292.69it/s, Materializing param=model.layers.14.mlp.up_proj.weight] Loading weights: 51%|█████▏ | 160/311 [00:00<00:00, 292.69it/s, Materializing param=model.layers.14.mlp.up_proj.weight] Loading weights: 58%|█████▊ | 179/311 [00:00<00:00, 277.44it/s, Materializing param=model.layers.16.input_layernorm.weight] Loading weights: 58%|█████▊ | 179/311 [00:00<00:00, 277.44it/s, Materializing param=model.layers.16.input_layernorm.weight] Loading weights: 58%|█████▊ | 180/311 [00:00<00:00, 277.44it/s, Materializing param=model.layers.16.mlp.down_proj.weight] Loading weights: 58%|█████▊ | 180/311 [00:00<00:00, 277.44it/s, Materializing param=model.layers.16.mlp.down_proj.weight] Loading weights: 71%|███████▏ | 222/311 [00:00<00:00, 349.86it/s, Materializing param=model.layers.19.self_attn.v_proj.weight] Loading weights: 71%|███████▏ | 222/311 [00:00<00:00, 349.86it/s, Materializing param=model.layers.19.self_attn.v_proj.weight] Loading weights: 76%|███████▌ | 237/311 [00:00<00:00, 375.64it/s, Materializing param=model.layers.21.mlp.up_proj.weight] Loading weights: 76%|███████▌ | 237/311 [00:00<00:00, 375.64it/s, Materializing param=model.layers.21.mlp.up_proj.weight] Loading weights: 76%|███████▌ | 237/311 [00:00<00:00, 372.07it/s, Materializing param=model.layers.21.mlp.up_proj.weight] Loading weights: 76%|███████▌ | 237/311 [00:00<00:00, 372.07it/s, Materializing param=model.layers.21.mlp.up_proj.weight] Loading weights: 44%|████▍ | 137/311 [00:00<00:00, 256.55it/s, Materializing param=model.layers.12.mlp.gate_proj.weight] Loading weights: 44%|████▍ | 137/311 [00:00<00:00, 256.55it/s, Materializing param=model.layers.12.mlp.gate_proj.weight] Loading weights: 72%|███████▏ | 223/311 [00:00<00:00, 349.86it/s, Materializing param=model.layers.20.input_layernorm.weight] Loading weights: 72%|███████▏ | 223/311 [00:00<00:00, 349.86it/s, Materializing param=model.layers.20.input_layernorm.weight] Loading weights: 72%|███████▏ | 224/311 [00:00<00:00, 349.86it/s, Materializing param=model.layers.20.mlp.down_proj.weight] Loading weights: 72%|███████▏ | 224/311 [00:00<00:00, 349.86it/s, Materializing param=model.layers.20.mlp.down_proj.weight] Loading weights: 69%|██████▉ | 216/311 [00:00<00:00, 357.93it/s, Materializing param=model.layers.19.post_attention_layernorm.weight] Loading weights: 69%|██████▉ | 216/311 [00:00<00:00, 357.93it/s, Materializing param=model.layers.19.post_attention_layernorm.weight] Loading weights: 70%|██████▉ | 217/311 [00:00<00:00, 357.93it/s, Materializing param=model.layers.19.self_attn.k_norm.weight] Loading weights: 70%|██████▉ | 217/311 [00:00<00:00, 357.93it/s, Materializing param=model.layers.19.self_attn.k_norm.weight] Loading weights: 70%|███████ | 218/311 [00:00<00:00, 357.93it/s, Materializing param=model.layers.19.self_attn.k_proj.weight] Loading weights: 70%|███████ | 218/311 [00:00<00:00, 357.93it/s, Materializing param=model.layers.19.self_attn.k_proj.weight] Loading weights: 70%|███████ | 219/311 [00:00<00:00, 357.93it/s, Materializing param=model.layers.19.self_attn.o_proj.weight] Loading weights: 70%|███████ | 219/311 [00:00<00:00, 357.93it/s, Materializing param=model.layers.19.self_attn.o_proj.weight] Loading weights: 71%|███████ | 220/311 [00:00<00:00, 357.93it/s, Materializing param=model.layers.19.self_attn.q_norm.weight] Loading weights: 71%|███████ | 220/311 [00:00<00:00, 357.93it/s, Materializing param=model.layers.19.self_attn.q_norm.weight] Loading weights: 71%|███████ | 221/311 [00:00<00:00, 357.93it/s, Materializing param=model.layers.19.self_attn.q_proj.weight] Loading weights: 71%|███████ | 221/311 [00:00<00:00, 357.93it/s, Materializing param=model.layers.19.self_attn.q_proj.weight] Loading weights: 58%|█████▊ | 181/311 [00:00<00:00, 277.44it/s, Materializing param=model.layers.16.mlp.gate_proj.weight] Loading weights: 58%|█████▊ | 181/311 [00:00<00:00, 277.44it/s, Materializing param=model.layers.16.mlp.gate_proj.weight] Loading weights: 52%|█████▏ | 161/311 [00:00<00:00, 292.69it/s, Materializing param=model.layers.14.post_attention_layernorm.weight] Loading weights: 52%|█████▏ | 161/311 [00:00<00:00, 292.69it/s, Materializing param=model.layers.14.post_attention_layernorm.weight] Loading weights: 72%|███████▏ | 225/311 [00:00<00:00, 349.86it/s, Materializing param=model.layers.20.mlp.gate_proj.weight] Loading weights: 52%|█████▏ | 162/311 [00:00<00:00, 292.69it/s, Materializing param=model.layers.14.self_attn.k_norm.weight] Loading weights: 72%|███████▏ | 225/311 [00:00<00:00, 349.86it/s, Materializing param=model.layers.20.mlp.gate_proj.weight] Loading weights: 52%|█████▏ | 162/311 [00:00<00:00, 292.69it/s, Materializing param=model.layers.14.self_attn.k_norm.weight] Loading weights: 77%|███████▋ | 238/311 [00:00<00:00, 375.64it/s, Materializing param=model.layers.21.post_attention_layernorm.weight] Loading weights: 52%|█████▏ | 163/311 [00:00<00:00, 292.69it/s, Materializing param=model.layers.14.self_attn.k_proj.weight] Loading weights: 77%|███████▋ | 238/311 [00:00<00:00, 375.64it/s, Materializing param=model.layers.21.post_attention_layernorm.weight] Loading weights: 52%|█████▏ | 163/311 [00:00<00:00, 292.69it/s, Materializing param=model.layers.14.self_attn.k_proj.weight] Loading weights: 77%|███████▋ | 238/311 [00:00<00:00, 372.07it/s, Materializing param=model.layers.21.post_attention_layernorm.weight] Loading weights: 77%|███████▋ | 238/311 [00:00<00:00, 372.07it/s, Materializing param=model.layers.21.post_attention_layernorm.weight] Loading weights: 77%|███████▋ | 239/311 [00:00<00:00, 375.64it/s, Materializing param=model.layers.21.self_attn.k_norm.weight] Loading weights: 77%|███████▋ | 239/311 [00:00<00:00, 375.64it/s, Materializing param=model.layers.21.self_attn.k_norm.weight] Loading weights: 77%|███████▋ | 239/311 [00:00<00:00, 372.07it/s, Materializing param=model.layers.21.self_attn.k_norm.weight] Loading weights: 53%|█████▎ | 164/311 [00:00<00:00, 292.69it/s, Materializing param=model.layers.14.self_attn.o_proj.weight] Loading weights: 53%|█████▎ | 164/311 [00:00<00:00, 292.69it/s, Materializing param=model.layers.14.self_attn.o_proj.weight] Loading weights: 77%|███████▋ | 239/311 [00:00<00:00, 372.07it/s, Materializing param=model.layers.21.self_attn.k_norm.weight] Loading weights: 77%|███████▋ | 240/311 [00:00<00:00, 375.64it/s, Materializing param=model.layers.21.self_attn.k_proj.weight] Loading weights: 77%|███████▋ | 240/311 [00:00<00:00, 375.64it/s, Materializing param=model.layers.21.self_attn.k_proj.weight] Loading weights: 77%|███████▋ | 240/311 [00:00<00:00, 372.07it/s, Materializing param=model.layers.21.self_attn.k_proj.weight] Loading weights: 77%|███████▋ | 240/311 [00:00<00:00, 372.07it/s, Materializing param=model.layers.21.self_attn.k_proj.weight] Loading weights: 77%|███████▋ | 241/311 [00:00<00:00, 375.64it/s, Materializing param=model.layers.21.self_attn.o_proj.weight] Loading weights: 77%|███████▋ | 241/311 [00:00<00:00, 375.64it/s, Materializing param=model.layers.21.self_attn.o_proj.weight] Loading weights: 77%|███████▋ | 241/311 [00:00<00:00, 372.07it/s, Materializing param=model.layers.21.self_attn.o_proj.weight] Loading weights: 77%|███████▋ | 241/311 [00:00<00:00, 372.07it/s, Materializing param=model.layers.21.self_attn.o_proj.weight] Loading weights: 71%|███████▏ | 222/311 [00:00<00:00, 357.93it/s, Materializing param=model.layers.19.self_attn.v_proj.weight] Loading weights: 71%|███████▏ | 222/311 [00:00<00:00, 357.93it/s, Materializing param=model.layers.19.self_attn.v_proj.weight] Loading weights: 72%|███████▏ | 223/311 [00:00<00:00, 357.93it/s, Materializing param=model.layers.20.input_layernorm.weight] Loading weights: 72%|███████▏ | 223/311 [00:00<00:00, 357.93it/s, Materializing param=model.layers.20.input_layernorm.weight] Loading weights: 53%|█████▎ | 165/311 [00:00<00:00, 292.69it/s, Materializing param=model.layers.14.self_attn.q_norm.weight] Loading weights: 53%|█████▎ | 165/311 [00:00<00:00, 292.69it/s, Materializing param=model.layers.14.self_attn.q_norm.weight] Loading weights: 72%|███████▏ | 224/311 [00:00<00:00, 357.93it/s, Materializing param=model.layers.20.mlp.down_proj.weight] Loading weights: 72%|███████▏ | 224/311 [00:00<00:00, 357.93it/s, Materializing param=model.layers.20.mlp.down_proj.weight] Loading weights: 53%|█████▎ | 166/311 [00:00<00:00, 292.69it/s, Materializing param=model.layers.14.self_attn.q_proj.weight] Loading weights: 53%|█████▎ | 166/311 [00:00<00:00, 292.69it/s, Materializing param=model.layers.14.self_attn.q_proj.weight] Loading weights: 78%|███████▊ | 242/311 [00:00<00:00, 403.91it/s, Materializing param=model.layers.21.self_attn.o_proj.weight] Loading weights: 78%|███████▊ | 242/311 [00:00<00:00, 403.91it/s, Materializing param=model.layers.21.self_attn.q_norm.weight] Loading weights: 78%|███████▊ | 242/311 [00:00<00:00, 375.64it/s, Materializing param=model.layers.21.self_attn.q_norm.weight] Loading weights: 78%|███████▊ | 242/311 [00:00<00:00, 403.91it/s, Materializing param=model.layers.21.self_attn.q_norm.weight] Loading weights: 78%|███████▊ | 242/311 [00:00<00:00, 375.64it/s, Materializing param=model.layers.21.self_attn.q_norm.weight] Loading weights: 78%|███████▊ | 243/311 [00:00<00:00, 403.91it/s, Materializing param=model.layers.21.self_attn.q_proj.weight] Loading weights: 78%|███████▊ | 243/311 [00:00<00:00, 375.64it/s, Materializing param=model.layers.21.self_attn.q_proj.weight] Loading weights: 78%|███████▊ | 243/311 [00:00<00:00, 403.91it/s, Materializing param=model.layers.21.self_attn.q_proj.weight] Loading weights: 78%|███████▊ | 243/311 [00:00<00:00, 375.64it/s, Materializing param=model.layers.21.self_attn.q_proj.weight] Loading weights: 54%|█████▎ | 167/311 [00:00<00:00, 292.69it/s, Materializing param=model.layers.14.self_attn.v_proj.weight] Loading weights: 54%|█████▎ | 167/311 [00:00<00:00, 292.69it/s, Materializing param=model.layers.14.self_attn.v_proj.weight] Loading weights: 78%|███████▊ | 244/311 [00:00<00:00, 375.64it/s, Materializing param=model.layers.21.self_attn.v_proj.weight] Loading weights: 73%|███████▎ | 226/311 [00:00<00:00, 349.86it/s, Materializing param=model.layers.20.mlp.up_proj.weight] Loading weights: 78%|███████▊ | 244/311 [00:00<00:00, 375.64it/s, Materializing param=model.layers.21.self_attn.v_proj.weight] Loading weights: 73%|███████▎ | 226/311 [00:00<00:00, 349.86it/s, Materializing param=model.layers.20.mlp.up_proj.weight] Loading weights: 79%|███████▉ | 245/311 [00:00<00:00, 375.64it/s, Materializing param=model.layers.22.input_layernorm.weight] Loading weights: 79%|███████▉ | 245/311 [00:00<00:00, 375.64it/s, Materializing param=model.layers.22.input_layernorm.weight] Loading weights: 79%|███████▉ | 246/311 [00:00<00:00, 375.64it/s, Materializing param=model.layers.22.mlp.down_proj.weight] Loading weights: 79%|███████▉ | 246/311 [00:00<00:00, 375.64it/s, Materializing param=model.layers.22.mlp.down_proj.weight] Loading weights: 54%|█████▍ | 168/311 [00:00<00:00, 292.69it/s, Materializing param=model.layers.15.input_layernorm.weight] Loading weights: 54%|█████▍ | 168/311 [00:00<00:00, 292.69it/s, Materializing param=model.layers.15.input_layernorm.weight] Loading weights: 54%|█████▍ | 169/311 [00:00<00:00, 292.69it/s, Materializing param=model.layers.15.mlp.down_proj.weight] Loading weights: 54%|█████▍ | 169/311 [00:00<00:00, 292.69it/s, Materializing param=model.layers.15.mlp.down_proj.weight] Loading weights: 78%|███████▊ | 244/311 [00:00<00:00, 403.91it/s, Materializing param=model.layers.21.self_attn.v_proj.weight] Loading weights: 78%|███████▊ | 244/311 [00:00<00:00, 403.91it/s, Materializing param=model.layers.21.self_attn.v_proj.weight] Loading weights: 79%|███████▉ | 245/311 [00:00<00:00, 403.91it/s, Materializing param=model.layers.22.input_layernorm.weight] Loading weights: 79%|███████▉ | 245/311 [00:00<00:00, 403.91it/s, Materializing param=model.layers.22.input_layernorm.weight] Loading weights: 79%|███████▉ | 246/311 [00:00<00:00, 403.91it/s, Materializing param=model.layers.22.mlp.down_proj.weight] Loading weights: 79%|███████▉ | 246/311 [00:00<00:00, 403.91it/s, Materializing param=model.layers.22.mlp.down_proj.weight] Loading weights: 59%|█████▊ | 182/311 [00:00<00:00, 277.44it/s, Materializing param=model.layers.16.mlp.up_proj.weight] Loading weights: 59%|█████▊ | 182/311 [00:00<00:00, 277.44it/s, Materializing param=model.layers.16.mlp.up_proj.weight] Loading weights: 72%|███████▏ | 225/311 [00:00<00:00, 357.93it/s, Materializing param=model.layers.20.mlp.gate_proj.weight] Loading weights: 72%|███████▏ | 225/311 [00:00<00:00, 357.93it/s, Materializing param=model.layers.20.mlp.gate_proj.weight] Loading weights: 44%|████▍ | 138/311 [00:00<00:00, 256.55it/s, Materializing param=model.layers.12.mlp.up_proj.weight] Loading weights: 44%|████▍ | 138/311 [00:00<00:00, 256.55it/s, Materializing param=model.layers.12.mlp.up_proj.weight] Loading weights: 45%|████▍ | 139/311 [00:00<00:00, 256.55it/s, Materializing param=model.layers.12.post_attention_layernorm.weight] Loading weights: 45%|████▍ | 139/311 [00:00<00:00, 256.55it/s, Materializing param=model.layers.12.post_attention_layernorm.weight] Loading weights: 45%|████▌ | 140/311 [00:00<00:00, 256.55it/s, Materializing param=model.layers.12.self_attn.k_norm.weight] Loading weights: 45%|████▌ | 140/311 [00:00<00:00, 256.55it/s, Materializing param=model.layers.12.self_attn.k_norm.weight] Loading weights: 45%|████▌ | 141/311 [00:00<00:00, 256.55it/s, Materializing param=model.layers.12.self_attn.k_proj.weight] Loading weights: 45%|████▌ | 141/311 [00:00<00:00, 256.55it/s, Materializing param=model.layers.12.self_attn.k_proj.weight] Loading weights: 46%|████▌ | 142/311 [00:00<00:00, 256.55it/s, Materializing param=model.layers.12.self_attn.o_proj.weight] Loading weights: 46%|████▌ | 142/311 [00:00<00:00, 256.55it/s, Materializing param=model.layers.12.self_attn.o_proj.weight] Loading weights: 79%|███████▉ | 247/311 [00:00<00:00, 375.64it/s, Materializing param=model.layers.22.mlp.gate_proj.weight] Loading weights: 79%|███████▉ | 247/311 [00:00<00:00, 375.64it/s, Materializing param=model.layers.22.mlp.gate_proj.weight] Loading weights: 79%|███████▉ | 247/311 [00:00<00:00, 403.91it/s, Materializing param=model.layers.22.mlp.gate_proj.weight] Loading weights: 79%|███████▉ | 247/311 [00:00<00:00, 403.91it/s, Materializing param=model.layers.22.mlp.gate_proj.weight] Loading weights: 73%|███████▎ | 227/311 [00:00<00:00, 379.16it/s, Materializing param=model.layers.20.mlp.up_proj.weight] Loading weights: 46%|████▌ | 143/311 [00:00<00:00, 256.55it/s, Materializing param=model.layers.12.self_attn.q_norm.weight] Loading weights: 46%|████▌ | 143/311 [00:00<00:00, 256.55it/s, Materializing param=model.layers.12.self_attn.q_norm.weight] Loading weights: 73%|███████▎ | 227/311 [00:00<00:00, 379.16it/s, Materializing param=model.layers.20.post_attention_layernorm.weight] Loading weights: 46%|████▋ | 144/311 [00:00<00:00, 256.55it/s, Materializing param=model.layers.12.self_attn.q_proj.weight] Loading weights: 46%|████▋ | 144/311 [00:00<00:00, 256.55it/s, Materializing param=model.layers.12.self_attn.q_proj.weight] Loading weights: 73%|███████▎ | 227/311 [00:00<00:00, 379.16it/s, Materializing param=model.layers.20.post_attention_layernorm.weight] Loading weights: 73%|███████▎ | 228/311 [00:00<00:00, 379.16it/s, Materializing param=model.layers.20.self_attn.k_norm.weight] Loading weights: 73%|███████▎ | 228/311 [00:00<00:00, 379.16it/s, Materializing param=model.layers.20.self_attn.k_norm.weight] Loading weights: 74%|███████▎ | 229/311 [00:00<00:00, 379.16it/s, Materializing param=model.layers.20.self_attn.k_proj.weight] Loading weights: 74%|███████▎ | 229/311 [00:00<00:00, 379.16it/s, Materializing param=model.layers.20.self_attn.k_proj.weight] Loading weights: 74%|███████▍ | 230/311 [00:00<00:00, 379.16it/s, Materializing param=model.layers.20.self_attn.o_proj.weight] Loading weights: 55%|█████▍ | 170/311 [00:00<00:00, 292.69it/s, Materializing param=model.layers.15.mlp.gate_proj.weight] Loading weights: 55%|█████▍ | 170/311 [00:00<00:00, 292.69it/s, Materializing param=model.layers.15.mlp.gate_proj.weight] Loading weights: 74%|███████▍ | 230/311 [00:00<00:00, 379.16it/s, Materializing param=model.layers.20.self_attn.o_proj.weight] Loading weights: 47%|████▋ | 145/311 [00:00<00:00, 256.55it/s, Materializing param=model.layers.12.self_attn.v_proj.weight] Loading weights: 47%|████▋ | 145/311 [00:00<00:00, 256.55it/s, Materializing param=model.layers.12.self_attn.v_proj.weight] Loading weights: 59%|█████▉ | 183/311 [00:00<00:00, 277.44it/s, Materializing param=model.layers.16.post_attention_layernorm.weight] Loading weights: 74%|███████▍ | 231/311 [00:00<00:00, 379.16it/s, Materializing param=model.layers.20.self_attn.q_norm.weight] Loading weights: 73%|███████▎ | 226/311 [00:00<00:00, 357.93it/s, Materializing param=model.layers.20.mlp.up_proj.weight] Loading weights: 59%|█████▉ | 183/311 [00:00<00:00, 277.44it/s, Materializing param=model.layers.16.post_attention_layernorm.weight] Loading weights: 74%|███████▍ | 231/311 [00:00<00:00, 379.16it/s, Materializing param=model.layers.20.self_attn.q_norm.weight] Loading weights: 73%|███████▎ | 226/311 [00:00<00:00, 357.93it/s, Materializing param=model.layers.20.mlp.up_proj.weight] Loading weights: 59%|█████▉ | 184/311 [00:00<00:00, 277.44it/s, Materializing param=model.layers.16.self_attn.k_norm.weight] Loading weights: 59%|█████▉ | 184/311 [00:00<00:00, 277.44it/s, Materializing param=model.layers.16.self_attn.k_norm.weight] Loading weights: 59%|█████▉ | 185/311 [00:00<00:00, 277.44it/s, Materializing param=model.layers.16.self_attn.k_proj.weight] Loading weights: 59%|█████▉ | 185/311 [00:00<00:00, 277.44it/s, Materializing param=model.layers.16.self_attn.k_proj.weight] Loading weights: 47%|████▋ | 146/311 [00:00<00:00, 256.55it/s, Materializing param=model.layers.13.input_layernorm.weight] Loading weights: 47%|████▋ | 146/311 [00:00<00:00, 256.55it/s, Materializing param=model.layers.13.input_layernorm.weight] Loading weights: 60%|█████▉ | 186/311 [00:00<00:00, 277.44it/s, Materializing param=model.layers.16.self_attn.o_proj.weight] Loading weights: 60%|█████▉ | 186/311 [00:00<00:00, 277.44it/s, Materializing param=model.layers.16.self_attn.o_proj.weight] Loading weights: 47%|████▋ | 147/311 [00:00<00:00, 256.55it/s, Materializing param=model.layers.13.mlp.down_proj.weight] Loading weights: 47%|████▋ | 147/311 [00:00<00:00, 256.55it/s, Materializing param=model.layers.13.mlp.down_proj.weight] Loading weights: 80%|███████▉ | 248/311 [00:00<00:00, 375.64it/s, Materializing param=model.layers.22.mlp.up_proj.weight] Loading weights: 80%|███████▉ | 248/311 [00:00<00:00, 403.91it/s, Materializing param=model.layers.22.mlp.up_proj.weight] Loading weights: 80%|███████▉ | 248/311 [00:00<00:00, 375.64it/s, Materializing param=model.layers.22.mlp.up_proj.weight] Loading weights: 75%|███████▍ | 232/311 [00:00<00:00, 379.16it/s, Materializing param=model.layers.20.self_attn.q_proj.weight] Loading weights: 80%|███████▉ | 248/311 [00:00<00:00, 403.91it/s, Materializing param=model.layers.22.mlp.up_proj.weight] Loading weights: 75%|███████▍ | 232/311 [00:00<00:00, 379.16it/s, Materializing param=model.layers.20.self_attn.q_proj.weight] Loading weights: 75%|███████▍ | 233/311 [00:00<00:00, 379.16it/s, Materializing param=model.layers.20.self_attn.v_proj.weight] Loading weights: 75%|███████▍ | 233/311 [00:00<00:00, 379.16it/s, Materializing param=model.layers.20.self_attn.v_proj.weight] Loading weights: 60%|██████ | 187/311 [00:00<00:00, 309.59it/s, Materializing param=model.layers.16.self_attn.o_proj.weight] Loading weights: 75%|███████▌ | 234/311 [00:00<00:00, 379.16it/s, Materializing param=model.layers.21.input_layernorm.weight] Loading weights: 60%|██████ | 187/311 [00:00<00:00, 309.59it/s, Materializing param=model.layers.16.self_attn.q_norm.weight] Loading weights: 75%|███████▌ | 234/311 [00:00<00:00, 379.16it/s, Materializing param=model.layers.21.input_layernorm.weight] Loading weights: 60%|██████ | 187/311 [00:00<00:00, 309.59it/s, Materializing param=model.layers.16.self_attn.q_norm.weight] Loading weights: 60%|██████ | 188/311 [00:00<00:00, 309.59it/s, Materializing param=model.layers.16.self_attn.q_proj.weight] Loading weights: 76%|███████▌ | 235/311 [00:00<00:00, 379.16it/s, Materializing param=model.layers.21.mlp.down_proj.weight] Loading weights: 60%|██████ | 188/311 [00:00<00:00, 309.59it/s, Materializing param=model.layers.16.self_attn.q_proj.weight] Loading weights: 76%|███████▌ | 235/311 [00:00<00:00, 379.16it/s, Materializing param=model.layers.21.mlp.down_proj.weight] Loading weights: 61%|██████ | 189/311 [00:00<00:00, 309.59it/s, Materializing param=model.layers.16.self_attn.v_proj.weight] Loading weights: 61%|██████ | 189/311 [00:00<00:00, 309.59it/s, Materializing param=model.layers.16.self_attn.v_proj.weight] Loading weights: 55%|█████▍ | 171/311 [00:00<00:00, 292.69it/s, Materializing param=model.layers.15.mlp.up_proj.weight] Loading weights: 55%|█████▍ | 171/311 [00:00<00:00, 292.69it/s, Materializing param=model.layers.15.mlp.up_proj.weight] Loading weights: 48%|████▊ | 148/311 [00:00<00:00, 256.55it/s, Materializing param=model.layers.13.mlp.gate_proj.weight] Loading weights: 73%|███████▎ | 227/311 [00:00<00:00, 357.93it/s, Materializing param=model.layers.20.post_attention_layernorm.weight] Loading weights: 48%|████▊ | 148/311 [00:00<00:00, 256.55it/s, Materializing param=model.layers.13.mlp.gate_proj.weight] Loading weights: 61%|██████ | 190/311 [00:00<00:00, 309.59it/s, Materializing param=model.layers.17.input_layernorm.weight] Loading weights: 61%|██████ | 190/311 [00:00<00:00, 309.59it/s, Materializing param=model.layers.17.input_layernorm.weight] Loading weights: 73%|███████▎ | 227/311 [00:00<00:00, 357.93it/s, Materializing param=model.layers.20.post_attention_layernorm.weight] Loading weights: 73%|███████▎ | 228/311 [00:00<00:00, 357.93it/s, Materializing param=model.layers.20.self_attn.k_norm.weight] Loading weights: 73%|███████▎ | 228/311 [00:00<00:00, 357.93it/s, Materializing param=model.layers.20.self_attn.k_norm.weight] Loading weights: 61%|██████▏ | 191/311 [00:00<00:00, 309.59it/s, Materializing param=model.layers.17.mlp.down_proj.weight] Loading weights: 61%|██████▏ | 191/311 [00:00<00:00, 309.59it/s, Materializing param=model.layers.17.mlp.down_proj.weight] Loading weights: 74%|███████▎ | 229/311 [00:00<00:00, 357.93it/s, Materializing param=model.layers.20.self_attn.k_proj.weight] Loading weights: 74%|███████▎ | 229/311 [00:00<00:00, 357.93it/s, Materializing param=model.layers.20.self_attn.k_proj.weight] Loading weights: 74%|███████▍ | 230/311 [00:00<00:00, 357.93it/s, Materializing param=model.layers.20.self_attn.o_proj.weight] Loading weights: 74%|███████▍ | 230/311 [00:00<00:00, 357.93it/s, Materializing param=model.layers.20.self_attn.o_proj.weight] Loading weights: 80%|████████ | 249/311 [00:00<00:00, 375.64it/s, Materializing param=model.layers.22.post_attention_layernorm.weight] Loading weights: 80%|████████ | 249/311 [00:00<00:00, 375.64it/s, Materializing param=model.layers.22.post_attention_layernorm.weight] Loading weights: 80%|████████ | 249/311 [00:00<00:00, 403.91it/s, Materializing param=model.layers.22.post_attention_layernorm.weight] Loading weights: 80%|████████ | 250/311 [00:00<00:00, 375.64it/s, Materializing param=model.layers.22.self_attn.k_norm.weight] Loading weights: 80%|████████ | 250/311 [00:00<00:00, 375.64it/s, Materializing param=model.layers.22.self_attn.k_norm.weight] Loading weights: 80%|████████ | 249/311 [00:00<00:00, 403.91it/s, Materializing param=model.layers.22.post_attention_layernorm.weight] Loading weights: 80%|████████ | 250/311 [00:00<00:00, 403.91it/s, Materializing param=model.layers.22.self_attn.k_norm.weight] Loading weights: 80%|████████ | 250/311 [00:00<00:00, 403.91it/s, Materializing param=model.layers.22.self_attn.k_norm.weight] Loading weights: 76%|███████▌ | 236/311 [00:00<00:00, 379.16it/s, Materializing param=model.layers.21.mlp.gate_proj.weight] Loading weights: 81%|████████ | 251/311 [00:00<00:00, 403.91it/s, Materializing param=model.layers.22.self_attn.k_proj.weight] Loading weights: 81%|████████ | 251/311 [00:00<00:00, 375.64it/s, Materializing param=model.layers.22.self_attn.k_proj.weight] Loading weights: 76%|███████▌ | 236/311 [00:00<00:00, 379.16it/s, Materializing param=model.layers.21.mlp.gate_proj.weight] Loading weights: 81%|████████ | 251/311 [00:00<00:00, 403.91it/s, Materializing param=model.layers.22.self_attn.k_proj.weight] Loading weights: 81%|████████ | 251/311 [00:00<00:00, 375.64it/s, Materializing param=model.layers.22.self_attn.k_proj.weight] Loading weights: 81%|████████ | 252/311 [00:00<00:00, 375.64it/s, Materializing param=model.layers.22.self_attn.o_proj.weight] Loading weights: 81%|████████ | 252/311 [00:00<00:00, 375.64it/s, Materializing param=model.layers.22.self_attn.o_proj.weight] Loading weights: 81%|████████ | 252/311 [00:00<00:00, 403.91it/s, Materializing param=model.layers.22.self_attn.o_proj.weight] Loading weights: 81%|████████ | 252/311 [00:00<00:00, 403.91it/s, Materializing param=model.layers.22.self_attn.o_proj.weight] Loading weights: 74%|███████▍ | 231/311 [00:00<00:00, 357.93it/s, Materializing param=model.layers.20.self_attn.q_norm.weight] Loading weights: 74%|███████▍ | 231/311 [00:00<00:00, 357.93it/s, Materializing param=model.layers.20.self_attn.q_norm.weight] Loading weights: 75%|███████▍ | 232/311 [00:00<00:00, 357.93it/s, Materializing param=model.layers.20.self_attn.q_proj.weight] Loading weights: 81%|████████▏ | 253/311 [00:00<00:00, 375.64it/s, Materializing param=model.layers.22.self_attn.q_norm.weight] Loading weights: 75%|███████▍ | 232/311 [00:00<00:00, 357.93it/s, Materializing param=model.layers.20.self_attn.q_proj.weight] Loading weights: 81%|████████▏ | 253/311 [00:00<00:00, 375.64it/s, Materializing param=model.layers.22.self_attn.q_norm.weight] Loading weights: 82%|████████▏ | 254/311 [00:00<00:00, 375.64it/s, Materializing param=model.layers.22.self_attn.q_proj.weight] Loading weights: 75%|███████▍ | 233/311 [00:00<00:00, 357.93it/s, Materializing param=model.layers.20.self_attn.v_proj.weight] Loading weights: 82%|████████▏ | 254/311 [00:00<00:00, 375.64it/s, Materializing param=model.layers.22.self_attn.q_proj.weight] Loading weights: 75%|███████▍ | 233/311 [00:00<00:00, 357.93it/s, Materializing param=model.layers.20.self_attn.v_proj.weight] Loading weights: 55%|█████▌ | 172/311 [00:00<00:00, 292.69it/s, Materializing param=model.layers.15.post_attention_layernorm.weight] Loading weights: 55%|█████▌ | 172/311 [00:00<00:00, 292.69it/s, Materializing param=model.layers.15.post_attention_layernorm.weight] Loading weights: 75%|███████▌ | 234/311 [00:00<00:00, 357.93it/s, Materializing param=model.layers.21.input_layernorm.weight] Loading weights: 75%|███████▌ | 234/311 [00:00<00:00, 357.93it/s, Materializing param=model.layers.21.input_layernorm.weight] Loading weights: 56%|█████▌ | 173/311 [00:00<00:00, 292.69it/s, Materializing param=model.layers.15.self_attn.k_norm.weight] Loading weights: 56%|█████▌ | 173/311 [00:00<00:00, 292.69it/s, Materializing param=model.layers.15.self_attn.k_norm.weight] Loading weights: 76%|███████▌ | 235/311 [00:00<00:00, 357.93it/s, Materializing param=model.layers.21.mlp.down_proj.weight] Loading weights: 56%|█████▌ | 174/311 [00:00<00:00, 292.69it/s, Materializing param=model.layers.15.self_attn.k_proj.weight] Loading weights: 82%|████████▏ | 255/311 [00:00<00:00, 375.64it/s, Materializing param=model.layers.22.self_attn.v_proj.weight] Loading weights: 76%|███████▌ | 235/311 [00:00<00:00, 357.93it/s, Materializing param=model.layers.21.mlp.down_proj.weight] Loading weights: 56%|█████▌ | 174/311 [00:00<00:00, 292.69it/s, Materializing param=model.layers.15.self_attn.k_proj.weight] Loading weights: 82%|████████▏ | 255/311 [00:00<00:00, 375.64it/s, Materializing param=model.layers.22.self_attn.v_proj.weight] Loading weights: 48%|████▊ | 149/311 [00:00<00:00, 256.55it/s, Materializing param=model.layers.13.mlp.up_proj.weight] Loading weights: 62%|██████▏ | 192/311 [00:00<00:00, 309.59it/s, Materializing param=model.layers.17.mlp.gate_proj.weight] Loading weights: 48%|████▊ | 149/311 [00:00<00:00, 256.55it/s, Materializing param=model.layers.13.mlp.up_proj.weight] Loading weights: 62%|██████▏ | 192/311 [00:00<00:00, 309.59it/s, Materializing param=model.layers.17.mlp.gate_proj.weight] Loading weights: 56%|█████▋ | 175/311 [00:00<00:00, 292.69it/s, Materializing param=model.layers.15.self_attn.o_proj.weight] Loading weights: 56%|█████▋ | 175/311 [00:00<00:00, 292.69it/s, Materializing param=model.layers.15.self_attn.o_proj.weight] Loading weights: 81%|████████▏ | 253/311 [00:00<00:00, 403.91it/s, Materializing param=model.layers.22.self_attn.q_norm.weight] Loading weights: 81%|████████▏ | 253/311 [00:00<00:00, 403.91it/s, Materializing param=model.layers.22.self_attn.q_norm.weight] Loading weights: 82%|████████▏ | 254/311 [00:00<00:00, 403.91it/s, Materializing param=model.layers.22.self_attn.q_proj.weight] Loading weights: 82%|████████▏ | 254/311 [00:00<00:00, 403.91it/s, Materializing param=model.layers.22.self_attn.q_proj.weight] Loading weights: 82%|████████▏ | 256/311 [00:00<00:00, 375.64it/s, Materializing param=model.layers.23.input_layernorm.weight] Loading weights: 82%|████████▏ | 256/311 [00:00<00:00, 375.64it/s, Materializing param=model.layers.23.input_layernorm.weight] Loading weights: 82%|████████▏ | 255/311 [00:00<00:00, 403.91it/s, Materializing param=model.layers.22.self_attn.v_proj.weight] Loading weights: 82%|████████▏ | 255/311 [00:00<00:00, 403.91it/s, Materializing param=model.layers.22.self_attn.v_proj.weight] Loading weights: 83%|████████▎ | 257/311 [00:00<00:00, 375.64it/s, Materializing param=model.layers.23.mlp.down_proj.weight] Loading weights: 83%|████████▎ | 257/311 [00:00<00:00, 375.64it/s, Materializing param=model.layers.23.mlp.down_proj.weight] Loading weights: 82%|████████▏ | 256/311 [00:00<00:00, 403.91it/s, Materializing param=model.layers.23.input_layernorm.weight] Loading weights: 82%|████████▏ | 256/311 [00:00<00:00, 403.91it/s, Materializing param=model.layers.23.input_layernorm.weight] Loading weights: 83%|████████▎ | 257/311 [00:00<00:00, 403.91it/s, Materializing param=model.layers.23.mlp.down_proj.weight] Loading weights: 76%|███████▌ | 237/311 [00:00<00:00, 379.16it/s, Materializing param=model.layers.21.mlp.up_proj.weight] Loading weights: 83%|████████▎ | 257/311 [00:00<00:00, 403.91it/s, Materializing param=model.layers.23.mlp.down_proj.weight] Loading weights: 76%|███████▌ | 237/311 [00:00<00:00, 379.16it/s, Materializing param=model.layers.21.mlp.up_proj.weight] Loading weights: 57%|█████▋ | 176/311 [00:00<00:00, 292.69it/s, Materializing param=model.layers.15.self_attn.q_norm.weight] Loading weights: 57%|█████▋ | 176/311 [00:00<00:00, 292.69it/s, Materializing param=model.layers.15.self_attn.q_norm.weight] Loading weights: 57%|█████▋ | 177/311 [00:00<00:00, 292.69it/s, Materializing param=model.layers.15.self_attn.q_proj.weight] Loading weights: 57%|█████▋ | 177/311 [00:00<00:00, 292.69it/s, Materializing param=model.layers.15.self_attn.q_proj.weight] Loading weights: 57%|█████▋ | 178/311 [00:00<00:00, 292.69it/s, Materializing param=model.layers.15.self_attn.v_proj.weight] Loading weights: 57%|█████▋ | 178/311 [00:00<00:00, 292.69it/s, Materializing param=model.layers.15.self_attn.v_proj.weight] Loading weights: 76%|███████▌ | 236/311 [00:00<00:00, 357.93it/s, Materializing param=model.layers.21.mlp.gate_proj.weight] Loading weights: 76%|███████▌ | 236/311 [00:00<00:00, 357.93it/s, Materializing param=model.layers.21.mlp.gate_proj.weight] Loading weights: 58%|█████▊ | 179/311 [00:00<00:00, 292.69it/s, Materializing param=model.layers.16.input_layernorm.weight] Loading weights: 58%|█████▊ | 179/311 [00:00<00:00, 292.69it/s, Materializing param=model.layers.16.input_layernorm.weight] Loading weights: 58%|█████▊ | 180/311 [00:00<00:00, 292.69it/s, Materializing param=model.layers.16.mlp.down_proj.weight] Loading weights: 58%|█████▊ | 180/311 [00:00<00:00, 292.69it/s, Materializing param=model.layers.16.mlp.down_proj.weight] Loading weights: 83%|████████▎ | 258/311 [00:00<00:00, 403.91it/s, Materializing param=model.layers.23.mlp.gate_proj.weight] Loading weights: 83%|████████▎ | 258/311 [00:00<00:00, 403.91it/s, Materializing param=model.layers.23.mlp.gate_proj.weight] Loading weights: 48%|████▊ | 150/311 [00:00<00:00, 256.55it/s, Materializing param=model.layers.13.post_attention_layernorm.weight] Loading weights: 62%|██████▏ | 193/311 [00:00<00:00, 309.59it/s, Materializing param=model.layers.17.mlp.up_proj.weight] Loading weights: 48%|████▊ | 150/311 [00:00<00:00, 256.55it/s, Materializing param=model.layers.13.post_attention_layernorm.weight] Loading weights: 62%|██████▏ | 193/311 [00:00<00:00, 309.59it/s, Materializing param=model.layers.17.mlp.up_proj.weight] Loading weights: 77%|███████▋ | 238/311 [00:00<00:00, 379.16it/s, Materializing param=model.layers.21.post_attention_layernorm.weight] Loading weights: 49%|████▊ | 151/311 [00:00<00:00, 256.55it/s, Materializing param=model.layers.13.self_attn.k_norm.weight] Loading weights: 49%|████▊ | 151/311 [00:00<00:00, 256.55it/s, Materializing param=model.layers.13.self_attn.k_norm.weight] Loading weights: 49%|████▉ | 152/311 [00:00<00:00, 256.55it/s, Materializing param=model.layers.13.self_attn.k_proj.weight] Loading weights: 77%|███████▋ | 238/311 [00:00<00:00, 379.16it/s, Materializing param=model.layers.21.post_attention_layernorm.weight] Loading weights: 49%|████▉ | 152/311 [00:00<00:00, 256.55it/s, Materializing param=model.layers.13.self_attn.k_proj.weight] Loading weights: 77%|███████▋ | 239/311 [00:00<00:00, 379.16it/s, Materializing param=model.layers.21.self_attn.k_norm.weight] Loading weights: 77%|███████▋ | 239/311 [00:00<00:00, 379.16it/s, Materializing param=model.layers.21.self_attn.k_norm.weight] Loading weights: 77%|███████▋ | 240/311 [00:00<00:00, 379.16it/s, Materializing param=model.layers.21.self_attn.k_proj.weight] Loading weights: 77%|███████▋ | 240/311 [00:00<00:00, 379.16it/s, Materializing param=model.layers.21.self_attn.k_proj.weight] Loading weights: 77%|███████▋ | 241/311 [00:00<00:00, 379.16it/s, Materializing param=model.layers.21.self_attn.o_proj.weight] Loading weights: 77%|███████▋ | 241/311 [00:00<00:00, 379.16it/s, Materializing param=model.layers.21.self_attn.o_proj.weight] Loading weights: 49%|████▉ | 153/311 [00:00<00:00, 256.55it/s, Materializing param=model.layers.13.self_attn.o_proj.weight] Loading weights: 49%|████▉ | 153/311 [00:00<00:00, 256.55it/s, Materializing param=model.layers.13.self_attn.o_proj.weight] Loading weights: 50%|████▉ | 154/311 [00:00<00:00, 256.55it/s, Materializing param=model.layers.13.self_attn.q_norm.weight] Loading weights: 78%|███████▊ | 242/311 [00:00<00:00, 379.16it/s, Materializing param=model.layers.21.self_attn.q_norm.weight] Loading weights: 50%|████▉ | 154/311 [00:00<00:00, 256.55it/s, Materializing param=model.layers.13.self_attn.q_norm.weight] Loading weights: 76%|███████▌ | 237/311 [00:00<00:00, 357.93it/s, Materializing param=model.layers.21.mlp.up_proj.weight] Loading weights: 78%|███████▊ | 242/311 [00:00<00:00, 379.16it/s, Materializing param=model.layers.21.self_attn.q_norm.weight] Loading weights: 76%|███████▌ | 237/311 [00:00<00:00, 357.93it/s, Materializing param=model.layers.21.mlp.up_proj.weight] Loading weights: 78%|███████▊ | 243/311 [00:00<00:00, 379.16it/s, Materializing param=model.layers.21.self_attn.q_proj.weight] Loading weights: 78%|███████▊ | 243/311 [00:00<00:00, 379.16it/s, Materializing param=model.layers.21.self_attn.q_proj.weight] Loading weights: 50%|████▉ | 155/311 [00:00<00:00, 256.55it/s, Materializing param=model.layers.13.self_attn.q_proj.weight] Loading weights: 83%|████████▎ | 259/311 [00:00<00:00, 403.91it/s, Materializing param=model.layers.23.mlp.up_proj.weight] Loading weights: 83%|████████▎ | 259/311 [00:00<00:00, 403.91it/s, Materializing param=model.layers.23.mlp.up_proj.weight] Loading weights: 50%|████▉ | 155/311 [00:00<00:00, 256.55it/s, Materializing param=model.layers.13.self_attn.q_proj.weight] Loading weights: 58%|█████▊ | 181/311 [00:00<00:00, 292.69it/s, Materializing param=model.layers.16.mlp.gate_proj.weight] Loading weights: 58%|█████▊ | 181/311 [00:00<00:00, 292.69it/s, Materializing param=model.layers.16.mlp.gate_proj.weight] Loading weights: 50%|█████ | 156/311 [00:00<00:00, 256.55it/s, Materializing param=model.layers.13.self_attn.v_proj.weight] Loading weights: 50%|█████ | 156/311 [00:00<00:00, 256.55it/s, Materializing param=model.layers.13.self_attn.v_proj.weight] Loading weights: 50%|█████ | 157/311 [00:00<00:00, 256.55it/s, Materializing param=model.layers.14.input_layernorm.weight] Loading weights: 78%|███████▊ | 244/311 [00:00<00:00, 379.16it/s, Materializing param=model.layers.21.self_attn.v_proj.weight] Loading weights: 50%|█████ | 157/311 [00:00<00:00, 256.55it/s, Materializing param=model.layers.14.input_layernorm.weight] Loading weights: 78%|███████▊ | 244/311 [00:00<00:00, 379.16it/s, Materializing param=model.layers.21.self_attn.v_proj.weight] Loading weights: 51%|█████ | 158/311 [00:00<00:00, 256.55it/s, Materializing param=model.layers.14.mlp.down_proj.weight] Loading weights: 51%|█████ | 158/311 [00:00<00:00, 256.55it/s, Materializing param=model.layers.14.mlp.down_proj.weight] Loading weights: 79%|███████▉ | 245/311 [00:00<00:00, 379.16it/s, Materializing param=model.layers.22.input_layernorm.weight] Loading weights: 79%|███████▉ | 245/311 [00:00<00:00, 379.16it/s, Materializing param=model.layers.22.input_layernorm.weight] Loading weights: 79%|███████▉ | 246/311 [00:00<00:00, 379.16it/s, Materializing param=model.layers.22.mlp.down_proj.weight] Loading weights: 79%|███████▉ | 246/311 [00:00<00:00, 379.16it/s, Materializing param=model.layers.22.mlp.down_proj.weight] Loading weights: 62%|██████▏ | 194/311 [00:00<00:00, 309.59it/s, Materializing param=model.layers.17.post_attention_layernorm.weight] Loading weights: 62%|██████▏ | 194/311 [00:00<00:00, 309.59it/s, Materializing param=model.layers.17.post_attention_layernorm.weight] Loading weights: 63%|██████▎ | 195/311 [00:00<00:00, 309.59it/s, Materializing param=model.layers.17.self_attn.k_norm.weight] Loading weights: 63%|██████▎ | 195/311 [00:00<00:00, 309.59it/s, Materializing param=model.layers.17.self_attn.k_norm.weight] Loading weights: 63%|██████▎ | 196/311 [00:00<00:00, 309.59it/s, Materializing param=model.layers.17.self_attn.k_proj.weight] Loading weights: 63%|██████▎ | 196/311 [00:00<00:00, 309.59it/s, Materializing param=model.layers.17.self_attn.k_proj.weight] Loading weights: 63%|██████▎ | 197/311 [00:00<00:00, 309.59it/s, Materializing param=model.layers.17.self_attn.o_proj.weight] Loading weights: 63%|██████▎ | 197/311 [00:00<00:00, 309.59it/s, Materializing param=model.layers.17.self_attn.o_proj.weight] Loading weights: 83%|████████▎ | 258/311 [00:00<00:00, 375.64it/s, Materializing param=model.layers.23.mlp.gate_proj.weight] Loading weights: 84%|████████▎ | 260/311 [00:00<00:00, 403.91it/s, Materializing param=model.layers.23.post_attention_layernorm.weight] Loading weights: 84%|████████▎ | 260/311 [00:00<00:00, 403.91it/s, Materializing param=model.layers.23.post_attention_layernorm.weight] Loading weights: 83%|████████▎ | 258/311 [00:00<00:00, 375.64it/s, Materializing param=model.layers.23.mlp.gate_proj.weight] Loading weights: 84%|████████▍ | 261/311 [00:00<00:00, 403.91it/s, Materializing param=model.layers.23.self_attn.k_norm.weight] Loading weights: 83%|████████▎ | 259/311 [00:00<00:00, 375.64it/s, Materializing param=model.layers.23.mlp.up_proj.weight] Loading weights: 84%|████████▍ | 261/311 [00:00<00:00, 403.91it/s, Materializing param=model.layers.23.self_attn.k_norm.weight] Loading weights: 83%|████████▎ | 259/311 [00:00<00:00, 375.64it/s, Materializing param=model.layers.23.mlp.up_proj.weight] Loading weights: 84%|████████▎ | 260/311 [00:00<00:00, 375.64it/s, Materializing param=model.layers.23.post_attention_layernorm.weight] Loading weights: 84%|████████▎ | 260/311 [00:00<00:00, 375.64it/s, Materializing param=model.layers.23.post_attention_layernorm.weight] Loading weights: 84%|████████▍ | 262/311 [00:00<00:00, 403.91it/s, Materializing param=model.layers.23.self_attn.k_proj.weight] Loading weights: 84%|████████▍ | 261/311 [00:00<00:00, 375.64it/s, Materializing param=model.layers.23.self_attn.k_norm.weight] Loading weights: 84%|████████▍ | 262/311 [00:00<00:00, 403.91it/s, Materializing param=model.layers.23.self_attn.k_proj.weight] Loading weights: 84%|████████▍ | 261/311 [00:00<00:00, 375.64it/s, Materializing param=model.layers.23.self_attn.k_norm.weight] Loading weights: 84%|████████▍ | 262/311 [00:00<00:00, 375.64it/s, Materializing param=model.layers.23.self_attn.k_proj.weight] Loading weights: 77%|███████▋ | 238/311 [00:00<00:00, 357.93it/s, Materializing param=model.layers.21.post_attention_layernorm.weight] Loading weights: 84%|████████▍ | 262/311 [00:00<00:00, 375.64it/s, Materializing param=model.layers.23.self_attn.k_proj.weight] Loading weights: 77%|███████▋ | 238/311 [00:00<00:00, 357.93it/s, Materializing param=model.layers.21.post_attention_layernorm.weight] Loading weights: 85%|████████▍ | 263/311 [00:00<00:00, 403.91it/s, Materializing param=model.layers.23.self_attn.o_proj.weight] Loading weights: 85%|████████▍ | 263/311 [00:00<00:00, 403.91it/s, Materializing param=model.layers.23.self_attn.o_proj.weight] Loading weights: 85%|████████▍ | 263/311 [00:00<00:00, 375.64it/s, Materializing param=model.layers.23.self_attn.o_proj.weight] Loading weights: 77%|███████▋ | 239/311 [00:00<00:00, 357.93it/s, Materializing param=model.layers.21.self_attn.k_norm.weight] Loading weights: 85%|████████▍ | 263/311 [00:00<00:00, 375.64it/s, Materializing param=model.layers.23.self_attn.o_proj.weight] Loading weights: 77%|███████▋ | 239/311 [00:00<00:00, 357.93it/s, Materializing param=model.layers.21.self_attn.k_norm.weight] Loading weights: 77%|███████▋ | 240/311 [00:00<00:00, 357.93it/s, Materializing param=model.layers.21.self_attn.k_proj.weight] Loading weights: 77%|███████▋ | 240/311 [00:00<00:00, 357.93it/s, Materializing param=model.layers.21.self_attn.k_proj.weight] Loading weights: 64%|██████▎ | 198/311 [00:00<00:00, 309.59it/s, Materializing param=model.layers.17.self_attn.q_norm.weight] Loading weights: 64%|██████▎ | 198/311 [00:00<00:00, 309.59it/s, Materializing param=model.layers.17.self_attn.q_norm.weight] Loading weights: 77%|███████▋ | 241/311 [00:00<00:00, 357.93it/s, Materializing param=model.layers.21.self_attn.o_proj.weight] Loading weights: 77%|███████▋ | 241/311 [00:00<00:00, 357.93it/s, Materializing param=model.layers.21.self_attn.o_proj.weight] Loading weights: 85%|████████▍ | 264/311 [00:00<00:00, 403.91it/s, Materializing param=model.layers.23.self_attn.q_norm.weight] Loading weights: 85%|████████▍ | 264/311 [00:00<00:00, 403.91it/s, Materializing param=model.layers.23.self_attn.q_norm.weight] Loading weights: 85%|████████▍ | 264/311 [00:00<00:00, 375.64it/s, Materializing param=model.layers.23.self_attn.q_norm.weight] Loading weights: 85%|████████▍ | 264/311 [00:00<00:00, 375.64it/s, Materializing param=model.layers.23.self_attn.q_norm.weight] Loading weights: 51%|█████ | 159/311 [00:00<00:00, 306.36it/s, Materializing param=model.layers.14.mlp.down_proj.weight] Loading weights: 85%|████████▌ | 265/311 [00:00<00:00, 375.64it/s, Materializing param=model.layers.23.self_attn.q_proj.weight] Loading weights: 51%|█████ | 159/311 [00:00<00:00, 306.36it/s, Materializing param=model.layers.14.mlp.gate_proj.weight] Loading weights: 85%|████████▌ | 265/311 [00:00<00:00, 375.64it/s, Materializing param=model.layers.23.self_attn.q_proj.weight] Loading weights: 51%|█████ | 159/311 [00:00<00:00, 306.36it/s, Materializing param=model.layers.14.mlp.gate_proj.weight] Loading weights: 59%|█████▊ | 182/311 [00:00<00:00, 292.69it/s, Materializing param=model.layers.16.mlp.up_proj.weight] Loading weights: 59%|█████▊ | 182/311 [00:00<00:00, 292.69it/s, Materializing param=model.layers.16.mlp.up_proj.weight] Loading weights: 64%|██████▍ | 199/311 [00:00<00:00, 309.59it/s, Materializing param=model.layers.17.self_attn.q_proj.weight] Loading weights: 64%|██████▍ | 199/311 [00:00<00:00, 309.59it/s, Materializing param=model.layers.17.self_attn.q_proj.weight] Loading weights: 64%|██████▍ | 200/311 [00:00<00:00, 309.59it/s, Materializing param=model.layers.17.self_attn.v_proj.weight] Loading weights: 64%|██████▍ | 200/311 [00:00<00:00, 309.59it/s, Materializing param=model.layers.17.self_attn.v_proj.weight] Loading weights: 78%|███████▊ | 242/311 [00:00<00:00, 357.93it/s, Materializing param=model.layers.21.self_attn.q_norm.weight] Loading weights: 78%|███████▊ | 242/311 [00:00<00:00, 357.93it/s, Materializing param=model.layers.21.self_attn.q_norm.weight] Loading weights: 86%|████████▌ | 266/311 [00:00<00:00, 375.64it/s, Materializing param=model.layers.23.self_attn.v_proj.weight] Loading weights: 65%|██████▍ | 201/311 [00:00<00:00, 309.59it/s, Materializing param=model.layers.18.input_layernorm.weight] Loading weights: 65%|██████▍ | 201/311 [00:00<00:00, 309.59it/s, Materializing param=model.layers.18.input_layernorm.weight] Loading weights: 86%|████████▌ | 266/311 [00:00<00:00, 375.64it/s, Materializing param=model.layers.23.self_attn.v_proj.weight] Loading weights: 78%|███████▊ | 243/311 [00:00<00:00, 357.93it/s, Materializing param=model.layers.21.self_attn.q_proj.weight] Loading weights: 86%|████████▌ | 267/311 [00:00<00:00, 375.64it/s, Materializing param=model.layers.24.input_layernorm.weight] Loading weights: 65%|██████▍ | 202/311 [00:00<00:00, 309.59it/s, Materializing param=model.layers.18.mlp.down_proj.weight] Loading weights: 86%|████████▌ | 267/311 [00:00<00:00, 375.64it/s, Materializing param=model.layers.24.input_layernorm.weight] Loading weights: 78%|███████▊ | 243/311 [00:00<00:00, 357.93it/s, Materializing param=model.layers.21.self_attn.q_proj.weight] Loading weights: 85%|████████▌ | 265/311 [00:00<00:00, 403.91it/s, Materializing param=model.layers.23.self_attn.q_proj.weight] Loading weights: 65%|██████▍ | 202/311 [00:00<00:00, 309.59it/s, Materializing param=model.layers.18.mlp.down_proj.weight] Loading weights: 86%|████████▌ | 268/311 [00:00<00:00, 375.64it/s, Materializing param=model.layers.24.mlp.down_proj.weight] Loading weights: 85%|████████▌ | 265/311 [00:00<00:00, 403.91it/s, Materializing param=model.layers.23.self_attn.q_proj.weight] Loading weights: 86%|████████▌ | 268/311 [00:00<00:00, 375.64it/s, Materializing param=model.layers.24.mlp.down_proj.weight] Loading weights: 86%|████████▌ | 266/311 [00:00<00:00, 403.91it/s, Materializing param=model.layers.23.self_attn.v_proj.weight] Loading weights: 86%|████████▌ | 266/311 [00:00<00:00, 403.91it/s, Materializing param=model.layers.23.self_attn.v_proj.weight] Loading weights: 86%|████████▌ | 267/311 [00:00<00:00, 403.91it/s, Materializing param=model.layers.24.input_layernorm.weight] Loading weights: 86%|████████▌ | 267/311 [00:00<00:00, 403.91it/s, Materializing param=model.layers.24.input_layernorm.weight] Loading weights: 86%|████████▌ | 268/311 [00:00<00:00, 403.91it/s, Materializing param=model.layers.24.mlp.down_proj.weight] Loading weights: 86%|████████▌ | 268/311 [00:00<00:00, 403.91it/s, Materializing param=model.layers.24.mlp.down_proj.weight] Loading weights: 78%|███████▊ | 244/311 [00:00<00:00, 357.93it/s, Materializing param=model.layers.21.self_attn.v_proj.weight] Loading weights: 78%|███████▊ | 244/311 [00:00<00:00, 357.93it/s, Materializing param=model.layers.21.self_attn.v_proj.weight] Loading weights: 79%|███████▉ | 245/311 [00:00<00:00, 357.93it/s, Materializing param=model.layers.22.input_layernorm.weight] Loading weights: 79%|███████▉ | 245/311 [00:00<00:00, 357.93it/s, Materializing param=model.layers.22.input_layernorm.weight] Loading weights: 79%|███████▉ | 247/311 [00:00<00:00, 379.16it/s, Materializing param=model.layers.22.mlp.gate_proj.weight] Loading weights: 79%|███████▉ | 247/311 [00:00<00:00, 379.16it/s, Materializing param=model.layers.22.mlp.gate_proj.weight] Loading weights: 80%|███████▉ | 248/311 [00:00<00:00, 379.16it/s, Materializing param=model.layers.22.mlp.up_proj.weight] Loading weights: 80%|███████▉ | 248/311 [00:00<00:00, 379.16it/s, Materializing param=model.layers.22.mlp.up_proj.weight] Loading weights: 51%|█████▏ | 160/311 [00:00<00:00, 306.36it/s, Materializing param=model.layers.14.mlp.up_proj.weight] Loading weights: 51%|█████▏ | 160/311 [00:00<00:00, 306.36it/s, Materializing param=model.layers.14.mlp.up_proj.weight] Loading weights: 59%|█████▉ | 183/311 [00:00<00:00, 292.69it/s, Materializing param=model.layers.16.post_attention_layernorm.weight] Loading weights: 59%|█████▉ | 183/311 [00:00<00:00, 292.69it/s, Materializing param=model.layers.16.post_attention_layernorm.weight] Loading weights: 59%|█████▉ | 184/311 [00:00<00:00, 292.69it/s, Materializing param=model.layers.16.self_attn.k_norm.weight] Loading weights: 59%|█████▉ | 184/311 [00:00<00:00, 292.69it/s, Materializing param=model.layers.16.self_attn.k_norm.weight] Loading weights: 59%|█████▉ | 185/311 [00:00<00:00, 292.69it/s, Materializing param=model.layers.16.self_attn.k_proj.weight] Loading weights: 59%|█████▉ | 185/311 [00:00<00:00, 292.69it/s, Materializing param=model.layers.16.self_attn.k_proj.weight] Loading weights: 60%|█████▉ | 186/311 [00:00<00:00, 292.69it/s, Materializing param=model.layers.16.self_attn.o_proj.weight] Loading weights: 60%|█████▉ | 186/311 [00:00<00:00, 292.69it/s, Materializing param=model.layers.16.self_attn.o_proj.weight] Loading weights: 86%|████████▋ | 269/311 [00:00<00:00, 375.64it/s, Materializing param=model.layers.24.mlp.gate_proj.weight] Loading weights: 86%|████████▋ | 269/311 [00:00<00:00, 403.91it/s, Materializing param=model.layers.24.mlp.gate_proj.weight] Loading weights: 86%|████████▋ | 269/311 [00:00<00:00, 375.64it/s, Materializing param=model.layers.24.mlp.gate_proj.weight] Loading weights: 86%|████████▋ | 269/311 [00:00<00:00, 403.91it/s, Materializing param=model.layers.24.mlp.gate_proj.weight] Loading weights: 80%|████████ | 249/311 [00:00<00:00, 379.16it/s, Materializing param=model.layers.22.post_attention_layernorm.weight] Loading weights: 60%|██████ | 187/311 [00:00<00:00, 292.69it/s, Materializing param=model.layers.16.self_attn.q_norm.weight] Loading weights: 60%|██████ | 187/311 [00:00<00:00, 292.69it/s, Materializing param=model.layers.16.self_attn.q_norm.weight] Loading weights: 80%|████████ | 249/311 [00:00<00:00, 379.16it/s, Materializing param=model.layers.22.post_attention_layernorm.weight] Loading weights: 60%|██████ | 188/311 [00:00<00:00, 292.69it/s, Materializing param=model.layers.16.self_attn.q_proj.weight] Loading weights: 60%|██████ | 188/311 [00:00<00:00, 292.69it/s, Materializing param=model.layers.16.self_attn.q_proj.weight] Loading weights: 80%|████████ | 250/311 [00:00<00:00, 379.16it/s, Materializing param=model.layers.22.self_attn.k_norm.weight] Loading weights: 80%|████████ | 250/311 [00:00<00:00, 379.16it/s, Materializing param=model.layers.22.self_attn.k_norm.weight] Loading weights: 81%|████████ | 251/311 [00:00<00:00, 379.16it/s, Materializing param=model.layers.22.self_attn.k_proj.weight] Loading weights: 81%|████████ | 251/311 [00:00<00:00, 379.16it/s, Materializing param=model.layers.22.self_attn.k_proj.weight] Loading weights: 79%|███████▉ | 246/311 [00:00<00:00, 357.93it/s, Materializing param=model.layers.22.mlp.down_proj.weight] Loading weights: 81%|████████ | 252/311 [00:00<00:00, 379.16it/s, Materializing param=model.layers.22.self_attn.o_proj.weight] Loading weights: 79%|███████▉ | 246/311 [00:00<00:00, 357.93it/s, Materializing param=model.layers.22.mlp.down_proj.weight] Loading weights: 81%|████████ | 252/311 [00:00<00:00, 379.16it/s, Materializing param=model.layers.22.self_attn.o_proj.weight] Loading weights: 79%|███████▉ | 247/311 [00:00<00:00, 357.93it/s, Materializing param=model.layers.22.mlp.gate_proj.weight] Loading weights: 79%|███████▉ | 247/311 [00:00<00:00, 357.93it/s, Materializing param=model.layers.22.mlp.gate_proj.weight] Loading weights: 61%|██████ | 189/311 [00:00<00:00, 292.69it/s, Materializing param=model.layers.16.self_attn.v_proj.weight] Loading weights: 61%|██████ | 189/311 [00:00<00:00, 292.69it/s, Materializing param=model.layers.16.self_attn.v_proj.weight] Loading weights: 87%|████████▋ | 270/311 [00:00<00:00, 375.64it/s, Materializing param=model.layers.24.mlp.up_proj.weight] Loading weights: 87%|████████▋ | 270/311 [00:00<00:00, 403.91it/s, Materializing param=model.layers.24.mlp.up_proj.weight] Loading weights: 87%|████████▋ | 270/311 [00:00<00:00, 403.91it/s, Materializing param=model.layers.24.mlp.up_proj.weight] Loading weights: 87%|████████▋ | 270/311 [00:00<00:00, 375.64it/s, Materializing param=model.layers.24.mlp.up_proj.weight] Loading weights: 52%|█████▏ | 161/311 [00:00<00:00, 306.36it/s, Materializing param=model.layers.14.post_attention_layernorm.weight] Loading weights: 52%|█████▏ | 161/311 [00:00<00:00, 306.36it/s, Materializing param=model.layers.14.post_attention_layernorm.weight] Loading weights: 81%|████████▏ | 253/311 [00:00<00:00, 379.16it/s, Materializing param=model.layers.22.self_attn.q_norm.weight] Loading weights: 52%|█████▏ | 162/311 [00:00<00:00, 306.36it/s, Materializing param=model.layers.14.self_attn.k_norm.weight] Loading weights: 81%|████████▏ | 253/311 [00:00<00:00, 379.16it/s, Materializing param=model.layers.22.self_attn.q_norm.weight] Loading weights: 52%|█████▏ | 162/311 [00:00<00:00, 306.36it/s, Materializing param=model.layers.14.self_attn.k_norm.weight] Loading weights: 52%|█████▏ | 163/311 [00:00<00:00, 306.36it/s, Materializing param=model.layers.14.self_attn.k_proj.weight] Loading weights: 82%|████████▏ | 254/311 [00:00<00:00, 379.16it/s, Materializing param=model.layers.22.self_attn.q_proj.weight] Loading weights: 52%|█████▏ | 163/311 [00:00<00:00, 306.36it/s, Materializing param=model.layers.14.self_attn.k_proj.weight] Loading weights: 82%|████████▏ | 254/311 [00:00<00:00, 379.16it/s, Materializing param=model.layers.22.self_attn.q_proj.weight] Loading weights: 61%|██████ | 190/311 [00:00<00:00, 292.69it/s, Materializing param=model.layers.17.input_layernorm.weight] Loading weights: 61%|██████ | 190/311 [00:00<00:00, 292.69it/s, Materializing param=model.layers.17.input_layernorm.weight] Loading weights: 61%|██████▏ | 191/311 [00:00<00:00, 292.69it/s, Materializing param=model.layers.17.mlp.down_proj.weight] Loading weights: 61%|██████▏ | 191/311 [00:00<00:00, 292.69it/s, Materializing param=model.layers.17.mlp.down_proj.weight] Loading weights: 53%|█████▎ | 164/311 [00:00<00:00, 306.36it/s, Materializing param=model.layers.14.self_attn.o_proj.weight] Loading weights: 53%|█████▎ | 164/311 [00:00<00:00, 306.36it/s, Materializing param=model.layers.14.self_attn.o_proj.weight] Loading weights: 65%|██████▌ | 203/311 [00:00<00:00, 309.59it/s, Materializing param=model.layers.18.mlp.gate_proj.weight] Loading weights: 65%|██████▌ | 203/311 [00:00<00:00, 309.59it/s, Materializing param=model.layers.18.mlp.gate_proj.weight] Loading weights: 66%|██████▌ | 204/311 [00:00<00:00, 309.59it/s, Materializing param=model.layers.18.mlp.up_proj.weight] Loading weights: 66%|██████▌ | 204/311 [00:00<00:00, 309.59it/s, Materializing param=model.layers.18.mlp.up_proj.weight] Loading weights: 82%|████████▏ | 255/311 [00:00<00:00, 379.16it/s, Materializing param=model.layers.22.self_attn.v_proj.weight] Loading weights: 82%|████████▏ | 255/311 [00:00<00:00, 379.16it/s, Materializing param=model.layers.22.self_attn.v_proj.weight] Loading weights: 82%|████████▏ | 256/311 [00:00<00:00, 379.16it/s, Materializing param=model.layers.23.input_layernorm.weight] Loading weights: 82%|████████▏ | 256/311 [00:00<00:00, 379.16it/s, Materializing param=model.layers.23.input_layernorm.weight] Loading weights: 83%|████████▎ | 257/311 [00:00<00:00, 379.16it/s, Materializing param=model.layers.23.mlp.down_proj.weight] Loading weights: 83%|████████▎ | 257/311 [00:00<00:00, 379.16it/s, Materializing param=model.layers.23.mlp.down_proj.weight] Loading weights: 53%|█████▎ | 165/311 [00:00<00:00, 306.36it/s, Materializing param=model.layers.14.self_attn.q_norm.weight] Loading weights: 53%|█████▎ | 165/311 [00:00<00:00, 306.36it/s, Materializing param=model.layers.14.self_attn.q_norm.weight] Loading weights: 53%|█████▎ | 166/311 [00:00<00:00, 306.36it/s, Materializing param=model.layers.14.self_attn.q_proj.weight] Loading weights: 53%|█████▎ | 166/311 [00:00<00:00, 306.36it/s, Materializing param=model.layers.14.self_attn.q_proj.weight] Loading weights: 80%|███████▉ | 248/311 [00:00<00:00, 357.93it/s, Materializing param=model.layers.22.mlp.up_proj.weight] Loading weights: 80%|███████▉ | 248/311 [00:00<00:00, 357.93it/s, Materializing param=model.layers.22.mlp.up_proj.weight] Loading weights: 54%|█████▎ | 167/311 [00:00<00:00, 306.36it/s, Materializing param=model.layers.14.self_attn.v_proj.weight] Loading weights: 54%|█████▎ | 167/311 [00:00<00:00, 306.36it/s, Materializing param=model.layers.14.self_attn.v_proj.weight] Loading weights: 87%|████████▋ | 271/311 [00:00<00:00, 375.64it/s, Materializing param=model.layers.24.post_attention_layernorm.weight] Loading weights: 87%|████████▋ | 271/311 [00:00<00:00, 375.64it/s, Materializing param=model.layers.24.post_attention_layernorm.weight] Loading weights: 87%|████████▋ | 271/311 [00:00<00:00, 403.91it/s, Materializing param=model.layers.24.post_attention_layernorm.weight] Loading weights: 54%|█████▍ | 168/311 [00:00<00:00, 306.36it/s, Materializing param=model.layers.15.input_layernorm.weight] Loading weights: 54%|█████▍ | 168/311 [00:00<00:00, 306.36it/s, Materializing param=model.layers.15.input_layernorm.weight] Loading weights: 87%|████████▋ | 271/311 [00:00<00:00, 403.91it/s, Materializing param=model.layers.24.post_attention_layernorm.weight] Loading weights: 87%|████████▋ | 272/311 [00:00<00:00, 403.91it/s, Materializing param=model.layers.24.self_attn.k_norm.weight] Loading weights: 87%|████████▋ | 272/311 [00:00<00:00, 375.64it/s, Materializing param=model.layers.24.self_attn.k_norm.weight] Loading weights: 87%|████████▋ | 272/311 [00:00<00:00, 403.91it/s, Materializing param=model.layers.24.self_attn.k_norm.weight] Loading weights: 54%|█████▍ | 169/311 [00:00<00:00, 306.36it/s, Materializing param=model.layers.15.mlp.down_proj.weight] Loading weights: 87%|████████▋ | 272/311 [00:00<00:00, 375.64it/s, Materializing param=model.layers.24.self_attn.k_norm.weight] Loading weights: 54%|█████▍ | 169/311 [00:00<00:00, 306.36it/s, Materializing param=model.layers.15.mlp.down_proj.weight] Loading weights: 88%|████████▊ | 273/311 [00:00<00:00, 403.91it/s, Materializing param=model.layers.24.self_attn.k_proj.weight] Loading weights: 88%|████████▊ | 273/311 [00:00<00:00, 375.64it/s, Materializing param=model.layers.24.self_attn.k_proj.weight] Loading weights: 88%|████████▊ | 273/311 [00:00<00:00, 403.91it/s, Materializing param=model.layers.24.self_attn.k_proj.weight] Loading weights: 88%|████████▊ | 273/311 [00:00<00:00, 375.64it/s, Materializing param=model.layers.24.self_attn.k_proj.weight] Loading weights: 88%|████████▊ | 274/311 [00:00<00:00, 403.91it/s, Materializing param=model.layers.24.self_attn.o_proj.weight] Loading weights: 88%|████████▊ | 274/311 [00:00<00:00, 375.64it/s, Materializing param=model.layers.24.self_attn.o_proj.weight] Loading weights: 88%|████████▊ | 274/311 [00:00<00:00, 403.91it/s, Materializing param=model.layers.24.self_attn.o_proj.weight] Loading weights: 88%|████████▊ | 274/311 [00:00<00:00, 375.64it/s, Materializing param=model.layers.24.self_attn.o_proj.weight] Loading weights: 62%|██████▏ | 192/311 [00:00<00:00, 292.69it/s, Materializing param=model.layers.17.mlp.gate_proj.weight] Loading weights: 62%|██████▏ | 192/311 [00:00<00:00, 292.69it/s, Materializing param=model.layers.17.mlp.gate_proj.weight] Loading weights: 66%|██████▌ | 205/311 [00:00<00:00, 309.59it/s, Materializing param=model.layers.18.post_attention_layernorm.weight] Loading weights: 66%|██████▌ | 205/311 [00:00<00:00, 309.59it/s, Materializing param=model.layers.18.post_attention_layernorm.weight] Loading weights: 66%|██████▌ | 206/311 [00:00<00:00, 309.59it/s, Materializing param=model.layers.18.self_attn.k_norm.weight] Loading weights: 66%|██████▌ | 206/311 [00:00<00:00, 309.59it/s, Materializing param=model.layers.18.self_attn.k_norm.weight] Loading weights: 67%|██████▋ | 207/311 [00:00<00:00, 309.59it/s, Materializing param=model.layers.18.self_attn.k_proj.weight] Loading weights: 67%|██████▋ | 207/311 [00:00<00:00, 309.59it/s, Materializing param=model.layers.18.self_attn.k_proj.weight] Loading weights: 67%|██████▋ | 208/311 [00:00<00:00, 309.59it/s, Materializing param=model.layers.18.self_attn.o_proj.weight] Loading weights: 67%|██████▋ | 208/311 [00:00<00:00, 309.59it/s, Materializing param=model.layers.18.self_attn.o_proj.weight] Loading weights: 83%|████████▎ | 258/311 [00:00<00:00, 379.16it/s, Materializing param=model.layers.23.mlp.gate_proj.weight] Loading weights: 88%|████████▊ | 275/311 [00:00<00:00, 403.91it/s, Materializing param=model.layers.24.self_attn.q_norm.weight] Loading weights: 83%|████████▎ | 258/311 [00:00<00:00, 379.16it/s, Materializing param=model.layers.23.mlp.gate_proj.weight] Loading weights: 88%|████████▊ | 275/311 [00:00<00:00, 406.45it/s, Materializing param=model.layers.24.self_attn.o_proj.weight] Loading weights: 88%|████████▊ | 275/311 [00:00<00:00, 403.91it/s, Materializing param=model.layers.24.self_attn.q_norm.weight] Loading weights: 88%|████████▊ | 275/311 [00:00<00:00, 406.45it/s, Materializing param=model.layers.24.self_attn.q_norm.weight] Loading weights: 88%|████████▊ | 275/311 [00:00<00:00, 406.45it/s, Materializing param=model.layers.24.self_attn.q_norm.weight] Loading weights: 89%|████████▊ | 276/311 [00:00<00:00, 403.91it/s, Materializing param=model.layers.24.self_attn.q_proj.weight] Loading weights: 89%|████████▊ | 276/311 [00:00<00:00, 403.91it/s, Materializing param=model.layers.24.self_attn.q_proj.weight] Loading weights: 89%|████████▊ | 276/311 [00:00<00:00, 406.45it/s, Materializing param=model.layers.24.self_attn.q_proj.weight] Loading weights: 89%|████████▊ | 276/311 [00:00<00:00, 406.45it/s, Materializing param=model.layers.24.self_attn.q_proj.weight] Loading weights: 80%|████████ | 249/311 [00:00<00:00, 357.93it/s, Materializing param=model.layers.22.post_attention_layernorm.weight] Loading weights: 80%|████████ | 249/311 [00:00<00:00, 357.93it/s, Materializing param=model.layers.22.post_attention_layernorm.weight] Loading weights: 80%|████████ | 250/311 [00:00<00:00, 357.93it/s, Materializing param=model.layers.22.self_attn.k_norm.weight] Loading weights: 80%|████████ | 250/311 [00:00<00:00, 357.93it/s, Materializing param=model.layers.22.self_attn.k_norm.weight] Loading weights: 81%|████████ | 251/311 [00:00<00:00, 357.93it/s, Materializing param=model.layers.22.self_attn.k_proj.weight] Loading weights: 81%|████████ | 251/311 [00:00<00:00, 357.93it/s, Materializing param=model.layers.22.self_attn.k_proj.weight] Loading weights: 89%|████████▉ | 277/311 [00:00<00:00, 403.91it/s, Materializing param=model.layers.24.self_attn.v_proj.weight] Loading weights: 67%|██████▋ | 209/311 [00:00<00:00, 309.59it/s, Materializing param=model.layers.18.self_attn.q_norm.weight] Loading weights: 89%|████████▉ | 277/311 [00:00<00:00, 403.91it/s, Materializing param=model.layers.24.self_attn.v_proj.weight] Loading weights: 67%|██████▋ | 209/311 [00:00<00:00, 309.59it/s, Materializing param=model.layers.18.self_attn.q_norm.weight] Loading weights: 68%|██████▊ | 210/311 [00:00<00:00, 309.59it/s, Materializing param=model.layers.18.self_attn.q_proj.weight] Loading weights: 89%|████████▉ | 277/311 [00:00<00:00, 406.45it/s, Materializing param=model.layers.24.self_attn.v_proj.weight] Loading weights: 68%|██████▊ | 210/311 [00:00<00:00, 309.59it/s, Materializing param=model.layers.18.self_attn.q_proj.weight] Loading weights: 89%|████████▉ | 277/311 [00:00<00:00, 406.45it/s, Materializing param=model.layers.24.self_attn.v_proj.weight] Loading weights: 81%|████████ | 252/311 [00:00<00:00, 357.93it/s, Materializing param=model.layers.22.self_attn.o_proj.weight] Loading weights: 81%|████████ | 252/311 [00:00<00:00, 357.93it/s, Materializing param=model.layers.22.self_attn.o_proj.weight] Loading weights: 89%|████████▉ | 278/311 [00:00<00:00, 406.45it/s, Materializing param=model.layers.25.input_layernorm.weight] Loading weights: 89%|████████▉ | 278/311 [00:00<00:00, 406.45it/s, Materializing param=model.layers.25.input_layernorm.weight] Loading weights: 90%|████████▉ | 279/311 [00:00<00:00, 406.45it/s, Materializing param=model.layers.25.mlp.down_proj.weight] Loading weights: 90%|████████▉ | 279/311 [00:00<00:00, 406.45it/s, Materializing param=model.layers.25.mlp.down_proj.weight] Loading weights: 89%|████████▉ | 278/311 [00:00<00:00, 403.91it/s, Materializing param=model.layers.25.input_layernorm.weight] Loading weights: 89%|████████▉ | 278/311 [00:00<00:00, 403.91it/s, Materializing param=model.layers.25.input_layernorm.weight] Loading weights: 90%|████████▉ | 279/311 [00:00<00:00, 403.91it/s, Materializing param=model.layers.25.mlp.down_proj.weight] Loading weights: 90%|████████▉ | 279/311 [00:00<00:00, 403.91it/s, Materializing param=model.layers.25.mlp.down_proj.weight] Loading weights: 68%|██████▊ | 211/311 [00:00<00:00, 309.59it/s, Materializing param=model.layers.18.self_attn.v_proj.weight] Loading weights: 68%|██████▊ | 211/311 [00:00<00:00, 309.59it/s, Materializing param=model.layers.18.self_attn.v_proj.weight] Loading weights: 55%|█████▍ | 170/311 [00:00<00:00, 306.36it/s, Materializing param=model.layers.15.mlp.gate_proj.weight] Loading weights: 55%|█████▍ | 170/311 [00:00<00:00, 306.36it/s, Materializing param=model.layers.15.mlp.gate_proj.weight] Loading weights: 81%|████████▏ | 253/311 [00:00<00:00, 357.93it/s, Materializing param=model.layers.22.self_attn.q_norm.weight] Loading weights: 81%|████████▏ | 253/311 [00:00<00:00, 357.93it/s, Materializing param=model.layers.22.self_attn.q_norm.weight] Loading weights: 68%|██████▊ | 212/311 [00:00<00:00, 309.59it/s, Materializing param=model.layers.19.input_layernorm.weight] Loading weights: 68%|██████▊ | 212/311 [00:00<00:00, 309.59it/s, Materializing param=model.layers.19.input_layernorm.weight] Loading weights: 62%|██████▏ | 193/311 [00:00<00:00, 315.03it/s, Materializing param=model.layers.17.mlp.gate_proj.weight] Loading weights: 62%|██████▏ | 193/311 [00:00<00:00, 315.03it/s, Materializing param=model.layers.17.mlp.up_proj.weight] Loading weights: 62%|██████▏ | 193/311 [00:00<00:00, 315.03it/s, Materializing param=model.layers.17.mlp.up_proj.weight] Loading weights: 83%|████████▎ | 259/311 [00:00<00:00, 379.16it/s, Materializing param=model.layers.23.mlp.up_proj.weight] Loading weights: 83%|████████▎ | 259/311 [00:00<00:00, 379.16it/s, Materializing param=model.layers.23.mlp.up_proj.weight] Loading weights: 82%|████████▏ | 254/311 [00:00<00:00, 357.93it/s, Materializing param=model.layers.22.self_attn.q_proj.weight] Loading weights: 82%|████████▏ | 254/311 [00:00<00:00, 357.93it/s, Materializing param=model.layers.22.self_attn.q_proj.weight] Loading weights: 82%|████████▏ | 255/311 [00:00<00:00, 357.93it/s, Materializing param=model.layers.22.self_attn.v_proj.weight] Loading weights: 82%|████████▏ | 255/311 [00:00<00:00, 357.93it/s, Materializing param=model.layers.22.self_attn.v_proj.weight] Loading weights: 90%|█████████ | 280/311 [00:00<00:00, 403.91it/s, Materializing param=model.layers.25.mlp.gate_proj.weight] Loading weights: 90%|█████████ | 280/311 [00:00<00:00, 403.91it/s, Materializing param=model.layers.25.mlp.gate_proj.weight] Loading weights: 82%|████████▏ | 256/311 [00:00<00:00, 357.93it/s, Materializing param=model.layers.23.input_layernorm.weight] Loading weights: 82%|████████▏ | 256/311 [00:00<00:00, 357.93it/s, Materializing param=model.layers.23.input_layernorm.weight] Loading weights: 83%|████████▎ | 257/311 [00:00<00:00, 357.93it/s, Materializing param=model.layers.23.mlp.down_proj.weight] Loading weights: 83%|████████▎ | 257/311 [00:00<00:00, 357.93it/s, Materializing param=model.layers.23.mlp.down_proj.weight] Loading weights: 55%|█████▍ | 171/311 [00:00<00:00, 306.36it/s, Materializing param=model.layers.15.mlp.up_proj.weight] Loading weights: 55%|█████▍ | 171/311 [00:00<00:00, 306.36it/s, Materializing param=model.layers.15.mlp.up_proj.weight] Loading weights: 90%|█████████ | 280/311 [00:00<00:00, 406.45it/s, Materializing param=model.layers.25.mlp.gate_proj.weight] Loading weights: 84%|████████▎ | 260/311 [00:00<00:00, 379.16it/s, Materializing param=model.layers.23.post_attention_layernorm.weight] Loading weights: 90%|█████████ | 280/311 [00:00<00:00, 406.45it/s, Materializing param=model.layers.25.mlp.gate_proj.weight] Loading weights: 84%|████████▎ | 260/311 [00:00<00:00, 379.16it/s, Materializing param=model.layers.23.post_attention_layernorm.weight] Loading weights: 68%|██████▊ | 213/311 [00:00<00:00, 309.59it/s, Materializing param=model.layers.19.mlp.down_proj.weight] Loading weights: 90%|█████████ | 281/311 [00:00<00:00, 406.45it/s, Materializing param=model.layers.25.mlp.up_proj.weight] Loading weights: 68%|██████▊ | 213/311 [00:00<00:00, 309.59it/s, Materializing param=model.layers.19.mlp.down_proj.weight] Loading weights: 90%|█████████ | 281/311 [00:00<00:00, 406.45it/s, Materializing param=model.layers.25.mlp.up_proj.weight] Loading weights: 69%|██████▉ | 214/311 [00:00<00:00, 309.59it/s, Materializing param=model.layers.19.mlp.gate_proj.weight] Loading weights: 69%|██████▉ | 214/311 [00:00<00:00, 309.59it/s, Materializing param=model.layers.19.mlp.gate_proj.weight] Loading weights: 84%|████████▍ | 261/311 [00:00<00:00, 379.16it/s, Materializing param=model.layers.23.self_attn.k_norm.weight] Loading weights: 62%|██████▏ | 194/311 [00:00<00:00, 315.03it/s, Materializing param=model.layers.17.post_attention_layernorm.weight] Loading weights: 84%|████████▍ | 261/311 [00:00<00:00, 379.16it/s, Materializing param=model.layers.23.self_attn.k_norm.weight] Loading weights: 84%|████████▍ | 262/311 [00:00<00:00, 379.16it/s, Materializing param=model.layers.23.self_attn.k_proj.weight] Loading weights: 62%|██████▏ | 194/311 [00:00<00:00, 315.03it/s, Materializing param=model.layers.17.post_attention_layernorm.weight] Loading weights: 84%|████████▍ | 262/311 [00:00<00:00, 379.16it/s, Materializing param=model.layers.23.self_attn.k_proj.weight] Loading weights: 63%|██████▎ | 195/311 [00:00<00:00, 315.03it/s, Materializing param=model.layers.17.self_attn.k_norm.weight] Loading weights: 63%|██████▎ | 195/311 [00:00<00:00, 315.03it/s, Materializing param=model.layers.17.self_attn.k_norm.weight] Loading weights: 63%|██████▎ | 196/311 [00:00<00:00, 315.03it/s, Materializing param=model.layers.17.self_attn.k_proj.weight] Loading weights: 63%|██████▎ | 196/311 [00:00<00:00, 315.03it/s, Materializing param=model.layers.17.self_attn.k_proj.weight] Loading weights: 85%|████████▍ | 263/311 [00:00<00:00, 379.16it/s, Materializing param=model.layers.23.self_attn.o_proj.weight] Loading weights: 85%|████████▍ | 263/311 [00:00<00:00, 379.16it/s, Materializing param=model.layers.23.self_attn.o_proj.weight] Loading weights: 63%|██████▎ | 197/311 [00:00<00:00, 315.03it/s, Materializing param=model.layers.17.self_attn.o_proj.weight] Loading weights: 63%|██████▎ | 197/311 [00:00<00:00, 315.03it/s, Materializing param=model.layers.17.self_attn.o_proj.weight] Loading weights: 90%|█████████ | 281/311 [00:00<00:00, 403.91it/s, Materializing param=model.layers.25.mlp.up_proj.weight] Loading weights: 90%|█████████ | 281/311 [00:00<00:00, 403.91it/s, Materializing param=model.layers.25.mlp.up_proj.weight] Loading weights: 85%|████████▍ | 264/311 [00:00<00:00, 379.16it/s, Materializing param=model.layers.23.self_attn.q_norm.weight] Loading weights: 85%|████████▍ | 264/311 [00:00<00:00, 379.16it/s, Materializing param=model.layers.23.self_attn.q_norm.weight] Loading weights: 64%|██████▎ | 198/311 [00:00<00:00, 315.03it/s, Materializing param=model.layers.17.self_attn.q_norm.weight] Loading weights: 64%|██████▎ | 198/311 [00:00<00:00, 315.03it/s, Materializing param=model.layers.17.self_attn.q_norm.weight] Loading weights: 64%|██████▍ | 199/311 [00:00<00:00, 315.03it/s, Materializing param=model.layers.17.self_attn.q_proj.weight] Loading weights: 64%|██████▍ | 199/311 [00:00<00:00, 315.03it/s, Materializing param=model.layers.17.self_attn.q_proj.weight] Loading weights: 85%|████████▌ | 265/311 [00:00<00:00, 379.16it/s, Materializing param=model.layers.23.self_attn.q_proj.weight] Loading weights: 85%|████████▌ | 265/311 [00:00<00:00, 379.16it/s, Materializing param=model.layers.23.self_attn.q_proj.weight] Loading weights: 86%|████████▌ | 266/311 [00:00<00:00, 379.16it/s, Materializing param=model.layers.23.self_attn.v_proj.weight] Loading weights: 55%|█████▌ | 172/311 [00:00<00:00, 306.36it/s, Materializing param=model.layers.15.post_attention_layernorm.weight] Loading weights: 86%|████████▌ | 266/311 [00:00<00:00, 379.16it/s, Materializing param=model.layers.23.self_attn.v_proj.weight] Loading weights: 55%|█████▌ | 172/311 [00:00<00:00, 306.36it/s, Materializing param=model.layers.15.post_attention_layernorm.weight] Loading weights: 91%|█████████ | 282/311 [00:00<00:00, 406.45it/s, Materializing param=model.layers.25.post_attention_layernorm.weight] Loading weights: 56%|█████▌ | 173/311 [00:00<00:00, 306.36it/s, Materializing param=model.layers.15.self_attn.k_norm.weight] Loading weights: 91%|█████████ | 282/311 [00:00<00:00, 406.45it/s, Materializing param=model.layers.25.post_attention_layernorm.weight] Loading weights: 56%|█████▌ | 173/311 [00:00<00:00, 306.36it/s, Materializing param=model.layers.15.self_attn.k_norm.weight] Loading weights: 56%|█████▌ | 174/311 [00:00<00:00, 306.36it/s, Materializing param=model.layers.15.self_attn.k_proj.weight] Loading weights: 56%|█████▌ | 174/311 [00:00<00:00, 306.36it/s, Materializing param=model.layers.15.self_attn.k_proj.weight] Loading weights: 86%|████████▌ | 267/311 [00:00<00:00, 379.16it/s, Materializing param=model.layers.24.input_layernorm.weight] Loading weights: 69%|██████▉ | 215/311 [00:00<00:00, 309.59it/s, Materializing param=model.layers.19.mlp.up_proj.weight] Loading weights: 86%|████████▌ | 267/311 [00:00<00:00, 379.16it/s, Materializing param=model.layers.24.input_layernorm.weight] Loading weights: 91%|█████████ | 283/311 [00:00<00:00, 406.45it/s, Materializing param=model.layers.25.self_attn.k_norm.weight] Loading weights: 69%|██████▉ | 215/311 [00:00<00:00, 309.59it/s, Materializing param=model.layers.19.mlp.up_proj.weight] Loading weights: 91%|█████████ | 283/311 [00:00<00:00, 406.45it/s, Materializing param=model.layers.25.self_attn.k_norm.weight] Loading weights: 91%|█████████ | 282/311 [00:00<00:00, 403.91it/s, Materializing param=model.layers.25.post_attention_layernorm.weight] Loading weights: 86%|████████▌ | 268/311 [00:00<00:00, 379.16it/s, Materializing param=model.layers.24.mlp.down_proj.weight] Loading weights: 91%|█████████ | 282/311 [00:00<00:00, 403.91it/s, Materializing param=model.layers.25.post_attention_layernorm.weight] Loading weights: 86%|████████▌ | 268/311 [00:00<00:00, 379.16it/s, Materializing param=model.layers.24.mlp.down_proj.weight] Loading weights: 56%|█████▋ | 175/311 [00:00<00:00, 306.36it/s, Materializing param=model.layers.15.self_attn.o_proj.weight] Loading weights: 91%|█████████▏| 284/311 [00:00<00:00, 406.45it/s, Materializing param=model.layers.25.self_attn.k_proj.weight] Loading weights: 91%|█████████ | 283/311 [00:00<00:00, 403.91it/s, Materializing param=model.layers.25.self_attn.k_norm.weight] Loading weights: 56%|█████▋ | 175/311 [00:00<00:00, 306.36it/s, Materializing param=model.layers.15.self_attn.o_proj.weight] Loading weights: 91%|█████████▏| 284/311 [00:00<00:00, 406.45it/s, Materializing param=model.layers.25.self_attn.k_proj.weight] Loading weights: 91%|█████████ | 283/311 [00:00<00:00, 403.91it/s, Materializing param=model.layers.25.self_attn.k_norm.weight] Loading weights: 64%|██████▍ | 200/311 [00:00<00:00, 315.03it/s, Materializing param=model.layers.17.self_attn.v_proj.weight] Loading weights: 64%|██████▍ | 200/311 [00:00<00:00, 315.03it/s, Materializing param=model.layers.17.self_attn.v_proj.weight] Loading weights: 91%|█████████▏| 284/311 [00:00<00:00, 403.91it/s, Materializing param=model.layers.25.self_attn.k_proj.weight] Loading weights: 91%|█████████▏| 284/311 [00:00<00:00, 403.91it/s, Materializing param=model.layers.25.self_attn.k_proj.weight] Loading weights: 92%|█████████▏| 285/311 [00:00<00:00, 406.45it/s, Materializing param=model.layers.25.self_attn.o_proj.weight] Loading weights: 92%|█████████▏| 285/311 [00:00<00:00, 406.45it/s, Materializing param=model.layers.25.self_attn.o_proj.weight] Loading weights: 92%|█████████▏| 285/311 [00:00<00:00, 403.91it/s, Materializing param=model.layers.25.self_attn.o_proj.weight] Loading weights: 92%|█████████▏| 285/311 [00:00<00:00, 403.91it/s, Materializing param=model.layers.25.self_attn.o_proj.weight] Loading weights: 65%|██████▍ | 201/311 [00:00<00:00, 315.03it/s, Materializing param=model.layers.18.input_layernorm.weight] Loading weights: 65%|██████▍ | 201/311 [00:00<00:00, 315.03it/s, Materializing param=model.layers.18.input_layernorm.weight] Loading weights: 65%|██████▍ | 202/311 [00:00<00:00, 315.03it/s, Materializing param=model.layers.18.mlp.down_proj.weight] Loading weights: 65%|██████▍ | 202/311 [00:00<00:00, 315.03it/s, Materializing param=model.layers.18.mlp.down_proj.weight] Loading weights: 92%|█████████▏| 286/311 [00:00<00:00, 403.91it/s, Materializing param=model.layers.25.self_attn.q_norm.weight] Loading weights: 92%|█████████▏| 286/311 [00:00<00:00, 403.91it/s, Materializing param=model.layers.25.self_attn.q_norm.weight] Loading weights: 92%|█████████▏| 286/311 [00:00<00:00, 406.45it/s, Materializing param=model.layers.25.self_attn.q_norm.weight] Loading weights: 92%|█████████▏| 286/311 [00:00<00:00, 406.45it/s, Materializing param=model.layers.25.self_attn.q_norm.weight] Loading weights: 92%|█████████▏| 287/311 [00:00<00:00, 403.91it/s, Materializing param=model.layers.25.self_attn.q_proj.weight] Loading weights: 92%|█████████▏| 287/311 [00:00<00:00, 403.91it/s, Materializing param=model.layers.25.self_attn.q_proj.weight] Loading weights: 92%|█████████▏| 287/311 [00:00<00:00, 406.45it/s, Materializing param=model.layers.25.self_attn.q_proj.weight] Loading weights: 92%|█████████▏| 287/311 [00:00<00:00, 406.45it/s, Materializing param=model.layers.25.self_attn.q_proj.weight] Loading weights: 83%|████████▎ | 258/311 [00:00<00:00, 369.38it/s, Materializing param=model.layers.23.mlp.down_proj.weight] Loading weights: 83%|████████▎ | 258/311 [00:00<00:00, 369.38it/s, Materializing param=model.layers.23.mlp.gate_proj.weight] Loading weights: 83%|████████▎ | 258/311 [00:00<00:00, 369.38it/s, Materializing param=model.layers.23.mlp.gate_proj.weight] Loading weights: 83%|████████▎ | 259/311 [00:00<00:00, 369.38it/s, Materializing param=model.layers.23.mlp.up_proj.weight] Loading weights: 83%|████████▎ | 259/311 [00:00<00:00, 369.38it/s, Materializing param=model.layers.23.mlp.up_proj.weight] Loading weights: 57%|█████▋ | 176/311 [00:00<00:00, 306.36it/s, Materializing param=model.layers.15.self_attn.q_norm.weight] Loading weights: 57%|█████▋ | 176/311 [00:00<00:00, 306.36it/s, Materializing param=model.layers.15.self_attn.q_norm.weight] Loading weights: 93%|█████████▎| 288/311 [00:00<00:00, 403.91it/s, Materializing param=model.layers.25.self_attn.v_proj.weight] Loading weights: 93%|█████████▎| 288/311 [00:00<00:00, 403.91it/s, Materializing param=model.layers.25.self_attn.v_proj.weight] Loading weights: 57%|█████▋ | 177/311 [00:00<00:00, 306.36it/s, Materializing param=model.layers.15.self_attn.q_proj.weight] Loading weights: 57%|█████▋ | 177/311 [00:00<00:00, 306.36it/s, Materializing param=model.layers.15.self_attn.q_proj.weight] Loading weights: 93%|█████████▎| 288/311 [00:00<00:00, 406.45it/s, Materializing param=model.layers.25.self_attn.v_proj.weight] Loading weights: 93%|█████████▎| 288/311 [00:00<00:00, 406.45it/s, Materializing param=model.layers.25.self_attn.v_proj.weight] Loading weights: 57%|█████▋ | 178/311 [00:00<00:00, 306.36it/s, Materializing param=model.layers.15.self_attn.v_proj.weight] Loading weights: 57%|█████▋ | 178/311 [00:00<00:00, 306.36it/s, Materializing param=model.layers.15.self_attn.v_proj.weight] Loading weights: 93%|█████████▎| 289/311 [00:00<00:00, 406.45it/s, Materializing param=model.layers.26.input_layernorm.weight] Loading weights: 93%|█████████▎| 289/311 [00:00<00:00, 406.45it/s, Materializing param=model.layers.26.input_layernorm.weight] Loading weights: 93%|█████████▎| 290/311 [00:00<00:00, 406.45it/s, Materializing param=model.layers.26.mlp.down_proj.weight] Loading weights: 93%|█████████▎| 290/311 [00:00<00:00, 406.45it/s, Materializing param=model.layers.26.mlp.down_proj.weight] Loading weights: 93%|█████████▎| 289/311 [00:00<00:00, 403.91it/s, Materializing param=model.layers.26.input_layernorm.weight] Loading weights: 93%|█████████▎| 289/311 [00:00<00:00, 403.91it/s, Materializing param=model.layers.26.input_layernorm.weight] Loading weights: 58%|█████▊ | 179/311 [00:00<00:00, 306.36it/s, Materializing param=model.layers.16.input_layernorm.weight] Loading weights: 93%|█████████▎| 290/311 [00:00<00:00, 403.91it/s, Materializing param=model.layers.26.mlp.down_proj.weight] Loading weights: 86%|████████▋ | 269/311 [00:00<00:00, 379.16it/s, Materializing param=model.layers.24.mlp.gate_proj.weight] Loading weights: 58%|█████▊ | 179/311 [00:00<00:00, 306.36it/s, Materializing param=model.layers.16.input_layernorm.weight] Loading weights: 93%|█████████▎| 290/311 [00:00<00:00, 403.91it/s, Materializing param=model.layers.26.mlp.down_proj.weight] Loading weights: 86%|████████▋ | 269/311 [00:00<00:00, 379.16it/s, Materializing param=model.layers.24.mlp.gate_proj.weight] Loading weights: 58%|█████▊ | 180/311 [00:00<00:00, 306.36it/s, Materializing param=model.layers.16.mlp.down_proj.weight] Loading weights: 58%|█████▊ | 180/311 [00:00<00:00, 306.36it/s, Materializing param=model.layers.16.mlp.down_proj.weight] Loading weights: 69%|██████▉ | 216/311 [00:00<00:00, 309.59it/s, Materializing param=model.layers.19.post_attention_layernorm.weight] Loading weights: 69%|██████▉ | 216/311 [00:00<00:00, 309.59it/s, Materializing param=model.layers.19.post_attention_layernorm.weight] Loading weights: 70%|██████▉ | 217/311 [00:00<00:00, 309.59it/s, Materializing param=model.layers.19.self_attn.k_norm.weight] Loading weights: 70%|██████▉ | 217/311 [00:00<00:00, 309.59it/s, Materializing param=model.layers.19.self_attn.k_norm.weight] Loading weights: 70%|███████ | 218/311 [00:00<00:00, 309.59it/s, Materializing param=model.layers.19.self_attn.k_proj.weight] Loading weights: 70%|███████ | 218/311 [00:00<00:00, 309.59it/s, Materializing param=model.layers.19.self_attn.k_proj.weight] Loading weights: 70%|███████ | 219/311 [00:00<00:00, 309.59it/s, Materializing param=model.layers.19.self_attn.o_proj.weight] Loading weights: 70%|███████ | 219/311 [00:00<00:00, 309.59it/s, Materializing param=model.layers.19.self_attn.o_proj.weight] Loading weights: 65%|██████▌ | 203/311 [00:00<00:00, 315.03it/s, Materializing param=model.layers.18.mlp.gate_proj.weight] Loading weights: 65%|██████▌ | 203/311 [00:00<00:00, 315.03it/s, Materializing param=model.layers.18.mlp.gate_proj.weight] Loading weights: 84%|████████▎ | 260/311 [00:00<00:00, 369.38it/s, Materializing param=model.layers.23.post_attention_layernorm.weight] Loading weights: 84%|████████▎ | 260/311 [00:00<00:00, 369.38it/s, Materializing param=model.layers.23.post_attention_layernorm.weight] Loading weights: 84%|████████▍ | 261/311 [00:00<00:00, 369.38it/s, Materializing param=model.layers.23.self_attn.k_norm.weight] Loading weights: 84%|████████▍ | 261/311 [00:00<00:00, 369.38it/s, Materializing param=model.layers.23.self_attn.k_norm.weight] Loading weights: 84%|████████▍ | 262/311 [00:00<00:00, 369.38it/s, Materializing param=model.layers.23.self_attn.k_proj.weight] Loading weights: 84%|████████▍ | 262/311 [00:00<00:00, 369.38it/s, Materializing param=model.layers.23.self_attn.k_proj.weight] Loading weights: 71%|███████ | 220/311 [00:00<00:00, 309.59it/s, Materializing param=model.layers.19.self_attn.q_norm.weight] Loading weights: 85%|████████▍ | 263/311 [00:00<00:00, 369.38it/s, Materializing param=model.layers.23.self_attn.o_proj.weight] Loading weights: 71%|███████ | 220/311 [00:00<00:00, 309.59it/s, Materializing param=model.layers.19.self_attn.q_norm.weight] Loading weights: 85%|████████▍ | 263/311 [00:00<00:00, 369.38it/s, Materializing param=model.layers.23.self_attn.o_proj.weight] Loading weights: 71%|███████ | 221/311 [00:00<00:00, 309.59it/s, Materializing param=model.layers.19.self_attn.q_proj.weight] Loading weights: 71%|███████ | 221/311 [00:00<00:00, 309.59it/s, Materializing param=model.layers.19.self_attn.q_proj.weight] Loading weights: 87%|████████▋ | 270/311 [00:00<00:00, 379.16it/s, Materializing param=model.layers.24.mlp.up_proj.weight] Loading weights: 87%|████████▋ | 270/311 [00:00<00:00, 379.16it/s, Materializing param=model.layers.24.mlp.up_proj.weight] Loading weights: 71%|███████▏ | 222/311 [00:00<00:00, 309.59it/s, Materializing param=model.layers.19.self_attn.v_proj.weight] Loading weights: 71%|███████▏ | 222/311 [00:00<00:00, 309.59it/s, Materializing param=model.layers.19.self_attn.v_proj.weight] Loading weights: 85%|████████▍ | 264/311 [00:00<00:00, 369.38it/s, Materializing param=model.layers.23.self_attn.q_norm.weight] Loading weights: 85%|████████▍ | 264/311 [00:00<00:00, 369.38it/s, Materializing param=model.layers.23.self_attn.q_norm.weight] Loading weights: 58%|█████▊ | 181/311 [00:00<00:00, 306.36it/s, Materializing param=model.layers.16.mlp.gate_proj.weight] Loading weights: 58%|█████▊ | 181/311 [00:00<00:00, 306.36it/s, Materializing param=model.layers.16.mlp.gate_proj.weight] Loading weights: 94%|█████████▎| 291/311 [00:00<00:00, 424.06it/s, Materializing param=model.layers.26.mlp.down_proj.weight] Loading weights: 94%|█████████▎| 291/311 [00:00<00:00, 406.45it/s, Materializing param=model.layers.26.mlp.gate_proj.weight] Loading weights: 94%|█████████▎| 291/311 [00:00<00:00, 406.45it/s, Materializing param=model.layers.26.mlp.gate_proj.weight] Loading weights: 94%|█████████▎| 291/311 [00:00<00:00, 424.06it/s, Materializing param=model.layers.26.mlp.gate_proj.weight] Loading weights: 94%|█████████▎| 291/311 [00:00<00:00, 424.06it/s, Materializing param=model.layers.26.mlp.gate_proj.weight] Loading weights: 72%|███████▏ | 223/311 [00:00<00:00, 309.59it/s, Materializing param=model.layers.20.input_layernorm.weight] Loading weights: 72%|███████▏ | 223/311 [00:00<00:00, 309.59it/s, Materializing param=model.layers.20.input_layernorm.weight] Loading weights: 85%|████████▌ | 265/311 [00:00<00:00, 369.38it/s, Materializing param=model.layers.23.self_attn.q_proj.weight] Loading weights: 85%|████████▌ | 265/311 [00:00<00:00, 369.38it/s, Materializing param=model.layers.23.self_attn.q_proj.weight] Loading weights: 86%|████████▌ | 266/311 [00:00<00:00, 369.38it/s, Materializing param=model.layers.23.self_attn.v_proj.weight] Loading weights: 86%|████████▌ | 266/311 [00:00<00:00, 369.38it/s, Materializing param=model.layers.23.self_attn.v_proj.weight] Loading weights: 66%|██████▌ | 204/311 [00:00<00:00, 315.03it/s, Materializing param=model.layers.18.mlp.up_proj.weight] Loading weights: 86%|████████▌ | 267/311 [00:00<00:00, 369.38it/s, Materializing param=model.layers.24.input_layernorm.weight] Loading weights: 66%|██████▌ | 204/311 [00:00<00:00, 315.03it/s, Materializing param=model.layers.18.mlp.up_proj.weight] Loading weights: 86%|████████▌ | 267/311 [00:00<00:00, 369.38it/s, Materializing param=model.layers.24.input_layernorm.weight] Loading weights: 86%|████████▌ | 268/311 [00:00<00:00, 369.38it/s, Materializing param=model.layers.24.mlp.down_proj.weight] Loading weights: 86%|████████▌ | 268/311 [00:00<00:00, 369.38it/s, Materializing param=model.layers.24.mlp.down_proj.weight] Loading weights: 87%|████████▋ | 271/311 [00:00<00:00, 379.16it/s, Materializing param=model.layers.24.post_attention_layernorm.weight] Loading weights: 87%|████████▋ | 271/311 [00:00<00:00, 379.16it/s, Materializing param=model.layers.24.post_attention_layernorm.weight] Loading weights: 87%|████████▋ | 272/311 [00:00<00:00, 400.33it/s, Materializing param=model.layers.24.post_attention_layernorm.weight] Loading weights: 87%|████████▋ | 272/311 [00:00<00:00, 400.33it/s, Materializing param=model.layers.24.self_attn.k_norm.weight] Loading weights: 87%|████████▋ | 272/311 [00:00<00:00, 400.33it/s, Materializing param=model.layers.24.self_attn.k_norm.weight] Loading weights: 88%|████████▊ | 273/311 [00:00<00:00, 400.33it/s, Materializing param=model.layers.24.self_attn.k_proj.weight] Loading weights: 88%|████████▊ | 273/311 [00:00<00:00, 400.33it/s, Materializing param=model.layers.24.self_attn.k_proj.weight] Loading weights: 94%|█████████▍| 292/311 [00:00<00:00, 406.45it/s, Materializing param=model.layers.26.mlp.up_proj.weight] Loading weights: 94%|█████████▍| 292/311 [00:00<00:00, 406.45it/s, Materializing param=model.layers.26.mlp.up_proj.weight] Loading weights: 94%|█████████▍| 292/311 [00:00<00:00, 424.06it/s, Materializing param=model.layers.26.mlp.up_proj.weight] Loading weights: 94%|█████████▍| 292/311 [00:00<00:00, 424.06it/s, Materializing param=model.layers.26.mlp.up_proj.weight] Loading weights: 59%|█████▊ | 182/311 [00:00<00:00, 306.36it/s, Materializing param=model.layers.16.mlp.up_proj.weight] Loading weights: 59%|█████▊ | 182/311 [00:00<00:00, 306.36it/s, Materializing param=model.layers.16.mlp.up_proj.weight] Loading weights: 88%|████████▊ | 274/311 [00:00<00:00, 400.33it/s, Materializing param=model.layers.24.self_attn.o_proj.weight] Loading weights: 88%|████████▊ | 274/311 [00:00<00:00, 400.33it/s, Materializing param=model.layers.24.self_attn.o_proj.weight] Loading weights: 88%|████████▊ | 275/311 [00:00<00:00, 400.33it/s, Materializing param=model.layers.24.self_attn.q_norm.weight] Loading weights: 88%|████████▊ | 275/311 [00:00<00:00, 400.33it/s, Materializing param=model.layers.24.self_attn.q_norm.weight] Loading weights: 89%|████████▊ | 276/311 [00:00<00:00, 400.33it/s, Materializing param=model.layers.24.self_attn.q_proj.weight] Loading weights: 89%|████████▊ | 276/311 [00:00<00:00, 400.33it/s, Materializing param=model.layers.24.self_attn.q_proj.weight] Loading weights: 86%|████████▋ | 269/311 [00:00<00:00, 369.38it/s, Materializing param=model.layers.24.mlp.gate_proj.weight] Loading weights: 86%|████████▋ | 269/311 [00:00<00:00, 369.38it/s, Materializing param=model.layers.24.mlp.gate_proj.weight] Loading weights: 89%|████████▉ | 277/311 [00:00<00:00, 400.33it/s, Materializing param=model.layers.24.self_attn.v_proj.weight] Loading weights: 72%|███████▏ | 224/311 [00:00<00:00, 309.59it/s, Materializing param=model.layers.20.mlp.down_proj.weight] Loading weights: 89%|████████▉ | 277/311 [00:00<00:00, 400.33it/s, Materializing param=model.layers.24.self_attn.v_proj.weight] Loading weights: 72%|███████▏ | 224/311 [00:00<00:00, 309.59it/s, Materializing param=model.layers.20.mlp.down_proj.weight] Loading weights: 72%|███████▏ | 225/311 [00:00<00:00, 309.59it/s, Materializing param=model.layers.20.mlp.gate_proj.weight] Loading weights: 72%|███████▏ | 225/311 [00:00<00:00, 309.59it/s, Materializing param=model.layers.20.mlp.gate_proj.weight] Loading weights: 66%|██████▌ | 205/311 [00:00<00:00, 315.03it/s, Materializing param=model.layers.18.post_attention_layernorm.weight] Loading weights: 66%|██████▌ | 205/311 [00:00<00:00, 315.03it/s, Materializing param=model.layers.18.post_attention_layernorm.weight] Loading weights: 94%|█████████▍| 293/311 [00:00<00:00, 406.45it/s, Materializing param=model.layers.26.post_attention_layernorm.weight] Loading weights: 94%|█████████▍| 293/311 [00:00<00:00, 406.45it/s, Materializing param=model.layers.26.post_attention_layernorm.weight] Loading weights: 66%|██████▌ | 206/311 [00:00<00:00, 315.03it/s, Materializing param=model.layers.18.self_attn.k_norm.weight] Loading weights: 66%|██████▌ | 206/311 [00:00<00:00, 315.03it/s, Materializing param=model.layers.18.self_attn.k_norm.weight] Loading weights: 89%|████████▉ | 278/311 [00:00<00:00, 400.33it/s, Materializing param=model.layers.25.input_layernorm.weight] Loading weights: 95%|█████████▍| 294/311 [00:00<00:00, 406.45it/s, Materializing param=model.layers.26.self_attn.k_norm.weight] Loading weights: 95%|█████████▍| 294/311 [00:00<00:00, 406.45it/s, Materializing param=model.layers.26.self_attn.k_norm.weight] Loading weights: 89%|████████▉ | 278/311 [00:00<00:00, 400.33it/s, Materializing param=model.layers.25.input_layernorm.weight] Loading weights: 67%|██████▋ | 207/311 [00:00<00:00, 315.03it/s, Materializing param=model.layers.18.self_attn.k_proj.weight] Loading weights: 67%|██████▋ | 207/311 [00:00<00:00, 315.03it/s, Materializing param=model.layers.18.self_attn.k_proj.weight] Loading weights: 95%|█████████▍| 295/311 [00:00<00:00, 406.45it/s, Materializing param=model.layers.26.self_attn.k_proj.weight] Loading weights: 95%|█████████▍| 295/311 [00:00<00:00, 406.45it/s, Materializing param=model.layers.26.self_attn.k_proj.weight] Loading weights: 95%|█████████▌| 296/311 [00:00<00:00, 406.45it/s, Materializing param=model.layers.26.self_attn.o_proj.weight] Loading weights: 95%|█████████▌| 296/311 [00:00<00:00, 406.45it/s, Materializing param=model.layers.26.self_attn.o_proj.weight] Loading weights: 94%|█████████▍| 293/311 [00:00<00:00, 424.06it/s, Materializing param=model.layers.26.post_attention_layernorm.weight] Loading weights: 94%|█████████▍| 293/311 [00:00<00:00, 424.06it/s, Materializing param=model.layers.26.post_attention_layernorm.weight] Loading weights: 95%|█████████▍| 294/311 [00:00<00:00, 424.06it/s, Materializing param=model.layers.26.self_attn.k_norm.weight] Loading weights: 95%|█████████▍| 294/311 [00:00<00:00, 424.06it/s, Materializing param=model.layers.26.self_attn.k_norm.weight] Loading weights: 95%|█████████▍| 295/311 [00:00<00:00, 424.06it/s, Materializing param=model.layers.26.self_attn.k_proj.weight] Loading weights: 95%|█████████▍| 295/311 [00:00<00:00, 424.06it/s, Materializing param=model.layers.26.self_attn.k_proj.weight] Loading weights: 95%|█████████▌| 296/311 [00:00<00:00, 424.06it/s, Materializing param=model.layers.26.self_attn.o_proj.weight] Loading weights: 95%|█████████▌| 296/311 [00:00<00:00, 424.06it/s, Materializing param=model.layers.26.self_attn.o_proj.weight] Loading weights: 59%|█████▉ | 183/311 [00:00<00:00, 306.36it/s, Materializing param=model.layers.16.post_attention_layernorm.weight] Loading weights: 59%|█████▉ | 183/311 [00:00<00:00, 306.36it/s, Materializing param=model.layers.16.post_attention_layernorm.weight] Loading weights: 59%|█████▉ | 184/311 [00:00<00:00, 306.36it/s, Materializing param=model.layers.16.self_attn.k_norm.weight] Loading weights: 59%|█████▉ | 184/311 [00:00<00:00, 306.36it/s, Materializing param=model.layers.16.self_attn.k_norm.weight] Loading weights: 59%|█████▉ | 185/311 [00:00<00:00, 306.36it/s, Materializing param=model.layers.16.self_attn.k_proj.weight] Loading weights: 59%|█████▉ | 185/311 [00:00<00:00, 306.36it/s, Materializing param=model.layers.16.self_attn.k_proj.weight] Loading weights: 67%|██████▋ | 208/311 [00:00<00:00, 315.03it/s, Materializing param=model.layers.18.self_attn.o_proj.weight] Loading weights: 67%|██████▋ | 208/311 [00:00<00:00, 315.03it/s, Materializing param=model.layers.18.self_attn.o_proj.weight] Loading weights: 67%|██████▋ | 209/311 [00:00<00:00, 315.03it/s, Materializing param=model.layers.18.self_attn.q_norm.weight] Loading weights: 95%|█████████▌| 297/311 [00:00<00:00, 424.06it/s, Materializing param=model.layers.26.self_attn.q_norm.weight] Loading weights: 67%|██████▋ | 209/311 [00:00<00:00, 315.03it/s, Materializing param=model.layers.18.self_attn.q_norm.weight] Loading weights: 95%|█████████▌| 297/311 [00:00<00:00, 424.06it/s, Materializing param=model.layers.26.self_attn.q_norm.weight] Loading weights: 68%|██████▊ | 210/311 [00:00<00:00, 315.03it/s, Materializing param=model.layers.18.self_attn.q_proj.weight] Loading weights: 60%|█████▉ | 186/311 [00:00<00:00, 306.36it/s, Materializing param=model.layers.16.self_attn.o_proj.weight] Loading weights: 96%|█████████▌| 298/311 [00:00<00:00, 424.06it/s, Materializing param=model.layers.26.self_attn.q_proj.weight] Loading weights: 68%|██████▊ | 210/311 [00:00<00:00, 315.03it/s, Materializing param=model.layers.18.self_attn.q_proj.weight] Loading weights: 60%|█████▉ | 186/311 [00:00<00:00, 306.36it/s, Materializing param=model.layers.16.self_attn.o_proj.weight] Loading weights: 96%|█████████▌| 298/311 [00:00<00:00, 424.06it/s, Materializing param=model.layers.26.self_attn.q_proj.weight] Loading weights: 96%|█████████▌| 299/311 [00:00<00:00, 424.06it/s, Materializing param=model.layers.26.self_attn.v_proj.weight] Loading weights: 96%|█████████▌| 299/311 [00:00<00:00, 424.06it/s, Materializing param=model.layers.26.self_attn.v_proj.weight] Loading weights: 73%|███████▎ | 226/311 [00:00<00:00, 327.90it/s, Materializing param=model.layers.20.mlp.gate_proj.weight] Loading weights: 96%|█████████▋| 300/311 [00:00<00:00, 424.06it/s, Materializing param=model.layers.27.input_layernorm.weight] Loading weights: 87%|████████▋ | 270/311 [00:00<00:00, 369.38it/s, Materializing param=model.layers.24.mlp.up_proj.weight] Loading weights: 96%|█████████▋| 300/311 [00:00<00:00, 424.06it/s, Materializing param=model.layers.27.input_layernorm.weight] Loading weights: 68%|██████▊ | 211/311 [00:00<00:00, 315.03it/s, Materializing param=model.layers.18.self_attn.v_proj.weight] Loading weights: 73%|███████▎ | 226/311 [00:00<00:00, 327.90it/s, Materializing param=model.layers.20.mlp.up_proj.weight] Loading weights: 68%|██████▊ | 211/311 [00:00<00:00, 315.03it/s, Materializing param=model.layers.18.self_attn.v_proj.weight] Loading weights: 73%|███████▎ | 226/311 [00:00<00:00, 327.90it/s, Materializing param=model.layers.20.mlp.up_proj.weight] Loading weights: 87%|████████▋ | 270/311 [00:00<00:00, 369.38it/s, Materializing param=model.layers.24.mlp.up_proj.weight] Loading weights: 90%|████████▉ | 279/311 [00:00<00:00, 400.33it/s, Materializing param=model.layers.25.mlp.down_proj.weight] Loading weights: 60%|██████ | 187/311 [00:00<00:00, 306.36it/s, Materializing param=model.layers.16.self_attn.q_norm.weight] Loading weights: 90%|████████▉ | 279/311 [00:00<00:00, 400.33it/s, Materializing param=model.layers.25.mlp.down_proj.weight] Loading weights: 60%|██████ | 187/311 [00:00<00:00, 306.36it/s, Materializing param=model.layers.16.self_attn.q_norm.weight] Loading weights: 95%|█████████▌| 297/311 [00:00<00:00, 406.45it/s, Materializing param=model.layers.26.self_attn.q_norm.weight] Loading weights: 60%|██████ | 188/311 [00:00<00:00, 306.36it/s, Materializing param=model.layers.16.self_attn.q_proj.weight] Loading weights: 95%|█████████▌| 297/311 [00:00<00:00, 406.45it/s, Materializing param=model.layers.26.self_attn.q_norm.weight] Loading weights: 90%|█████████ | 280/311 [00:00<00:00, 400.33it/s, Materializing param=model.layers.25.mlp.gate_proj.weight] Loading weights: 60%|██████ | 188/311 [00:00<00:00, 306.36it/s, Materializing param=model.layers.16.self_attn.q_proj.weight] Loading weights: 90%|█████████ | 280/311 [00:00<00:00, 400.33it/s, Materializing param=model.layers.25.mlp.gate_proj.weight] Loading weights: 96%|█████████▌| 298/311 [00:00<00:00, 406.45it/s, Materializing param=model.layers.26.self_attn.q_proj.weight] Loading weights: 96%|█████████▌| 298/311 [00:00<00:00, 406.45it/s, Materializing param=model.layers.26.self_attn.q_proj.weight] Loading weights: 96%|█████████▌| 299/311 [00:00<00:00, 406.45it/s, Materializing param=model.layers.26.self_attn.v_proj.weight] Loading weights: 96%|█████████▌| 299/311 [00:00<00:00, 406.45it/s, Materializing param=model.layers.26.self_attn.v_proj.weight] Loading weights: 96%|█████████▋| 300/311 [00:00<00:00, 406.45it/s, Materializing param=model.layers.27.input_layernorm.weight] Loading weights: 68%|██████▊ | 212/311 [00:00<00:00, 315.03it/s, Materializing param=model.layers.19.input_layernorm.weight] Loading weights: 96%|█████████▋| 300/311 [00:00<00:00, 406.45it/s, Materializing param=model.layers.27.input_layernorm.weight] Loading weights: 68%|██████▊ | 212/311 [00:00<00:00, 315.03it/s, Materializing param=model.layers.19.input_layernorm.weight] Loading weights: 97%|█████████▋| 301/311 [00:00<00:00, 406.45it/s, Materializing param=model.layers.27.mlp.down_proj.weight] Loading weights: 68%|██████▊ | 213/311 [00:00<00:00, 315.03it/s, Materializing param=model.layers.19.mlp.down_proj.weight] Loading weights: 97%|█████████▋| 301/311 [00:00<00:00, 406.45it/s, Materializing param=model.layers.27.mlp.down_proj.weight] Loading weights: 68%|██████▊ | 213/311 [00:00<00:00, 315.03it/s, Materializing param=model.layers.19.mlp.down_proj.weight] Loading weights: 61%|██████ | 189/311 [00:00<00:00, 306.36it/s, Materializing param=model.layers.16.self_attn.v_proj.weight] Loading weights: 61%|██████ | 189/311 [00:00<00:00, 306.36it/s, Materializing param=model.layers.16.self_attn.v_proj.weight] Loading weights: 61%|██████ | 190/311 [00:00<00:00, 306.36it/s, Materializing param=model.layers.17.input_layernorm.weight] Loading weights: 61%|██████ | 190/311 [00:00<00:00, 306.36it/s, Materializing param=model.layers.17.input_layernorm.weight] Loading weights: 61%|██████▏ | 191/311 [00:00<00:00, 306.36it/s, Materializing param=model.layers.17.mlp.down_proj.weight] Loading weights: 61%|██████▏ | 191/311 [00:00<00:00, 306.36it/s, Materializing param=model.layers.17.mlp.down_proj.weight] Loading weights: 97%|█████████▋| 302/311 [00:00<00:00, 406.45it/s, Materializing param=model.layers.27.mlp.gate_proj.weight] Loading weights: 97%|█████████▋| 302/311 [00:00<00:00, 406.45it/s, Materializing param=model.layers.27.mlp.gate_proj.weight] Loading weights: 97%|█████████▋| 301/311 [00:00<00:00, 424.06it/s, Materializing param=model.layers.27.mlp.down_proj.weight] Loading weights: 97%|█████████▋| 301/311 [00:00<00:00, 424.06it/s, Materializing param=model.layers.27.mlp.down_proj.weight] Loading weights: 97%|█████████▋| 302/311 [00:00<00:00, 424.06it/s, Materializing param=model.layers.27.mlp.gate_proj.weight] Loading weights: 73%|███████▎ | 227/311 [00:00<00:00, 327.90it/s, Materializing param=model.layers.20.post_attention_layernorm.weight] Loading weights: 97%|█████████▋| 302/311 [00:00<00:00, 424.06it/s, Materializing param=model.layers.27.mlp.gate_proj.weight] Loading weights: 73%|███████▎ | 227/311 [00:00<00:00, 327.90it/s, Materializing param=model.layers.20.post_attention_layernorm.weight] Loading weights: 73%|███████▎ | 228/311 [00:00<00:00, 327.90it/s, Materializing param=model.layers.20.self_attn.k_norm.weight] Loading weights: 73%|███████▎ | 228/311 [00:00<00:00, 327.90it/s, Materializing param=model.layers.20.self_attn.k_norm.weight] Loading weights: 74%|███████▎ | 229/311 [00:00<00:00, 327.90it/s, Materializing param=model.layers.20.self_attn.k_proj.weight] Loading weights: 90%|█████████ | 281/311 [00:00<00:00, 400.33it/s, Materializing param=model.layers.25.mlp.up_proj.weight] Loading weights: 74%|███████▎ | 229/311 [00:00<00:00, 327.90it/s, Materializing param=model.layers.20.self_attn.k_proj.weight] Loading weights: 90%|█████████ | 281/311 [00:00<00:00, 400.33it/s, Materializing param=model.layers.25.mlp.up_proj.weight] Loading weights: 87%|████████▋ | 271/311 [00:00<00:00, 369.38it/s, Materializing param=model.layers.24.post_attention_layernorm.weight] Loading weights: 74%|███████▍ | 230/311 [00:00<00:00, 327.90it/s, Materializing param=model.layers.20.self_attn.o_proj.weight] Loading weights: 87%|████████▋ | 271/311 [00:00<00:00, 369.38it/s, Materializing param=model.layers.24.post_attention_layernorm.weight] Loading weights: 74%|███████▍ | 230/311 [00:00<00:00, 327.90it/s, Materializing param=model.layers.20.self_attn.o_proj.weight] Loading weights: 87%|████████▋ | 272/311 [00:00<00:00, 369.38it/s, Materializing param=model.layers.24.self_attn.k_norm.weight] Loading weights: 87%|████████▋ | 272/311 [00:00<00:00, 369.38it/s, Materializing param=model.layers.24.self_attn.k_norm.weight] Loading weights: 88%|████████▊ | 273/311 [00:00<00:00, 369.38it/s, Materializing param=model.layers.24.self_attn.k_proj.weight] Loading weights: 88%|████████▊ | 273/311 [00:00<00:00, 369.38it/s, Materializing param=model.layers.24.self_attn.k_proj.weight] Loading weights: 69%|██████▉ | 214/311 [00:00<00:00, 315.03it/s, Materializing param=model.layers.19.mlp.gate_proj.weight] Loading weights: 69%|██████▉ | 214/311 [00:00<00:00, 315.03it/s, Materializing param=model.layers.19.mlp.gate_proj.weight] Loading weights: 88%|████████▊ | 274/311 [00:00<00:00, 369.38it/s, Materializing param=model.layers.24.self_attn.o_proj.weight] Loading weights: 88%|████████▊ | 274/311 [00:00<00:00, 369.38it/s, Materializing param=model.layers.24.self_attn.o_proj.weight] Loading weights: 74%|███████▍ | 231/311 [00:00<00:00, 327.90it/s, Materializing param=model.layers.20.self_attn.q_norm.weight] Loading weights: 74%|███████▍ | 231/311 [00:00<00:00, 327.90it/s, Materializing param=model.layers.20.self_attn.q_norm.weight] Loading weights: 62%|██████▏ | 192/311 [00:00<00:00, 306.36it/s, Materializing param=model.layers.17.mlp.gate_proj.weight] Loading weights: 62%|██████▏ | 192/311 [00:00<00:00, 306.36it/s, Materializing param=model.layers.17.mlp.gate_proj.weight] Loading weights: 88%|████████▊ | 275/311 [00:00<00:00, 369.38it/s, Materializing param=model.layers.24.self_attn.q_norm.weight] Loading weights: 88%|████████▊ | 275/311 [00:00<00:00, 369.38it/s, Materializing param=model.layers.24.self_attn.q_norm.weight] Loading weights: 89%|████████▊ | 276/311 [00:00<00:00, 369.38it/s, Materializing param=model.layers.24.self_attn.q_proj.weight] Loading weights: 89%|████████▊ | 276/311 [00:00<00:00, 369.38it/s, Materializing param=model.layers.24.self_attn.q_proj.weight] Loading weights: 75%|███████▍ | 232/311 [00:00<00:00, 327.90it/s, Materializing param=model.layers.20.self_attn.q_proj.weight] Loading weights: 75%|███████▍ | 232/311 [00:00<00:00, 327.90it/s, Materializing param=model.layers.20.self_attn.q_proj.weight] Loading weights: 75%|███████▍ | 233/311 [00:00<00:00, 327.90it/s, Materializing param=model.layers.20.self_attn.v_proj.weight] Loading weights: 75%|███████▍ | 233/311 [00:00<00:00, 327.90it/s, Materializing param=model.layers.20.self_attn.v_proj.weight] Loading weights: 97%|█████████▋| 303/311 [00:00<00:00, 406.45it/s, Materializing param=model.layers.27.mlp.up_proj.weight] Loading weights: 97%|█████████▋| 303/311 [00:00<00:00, 424.06it/s, Materializing param=model.layers.27.mlp.up_proj.weight] Loading weights: 89%|████████▉ | 277/311 [00:00<00:00, 369.38it/s, Materializing param=model.layers.24.self_attn.v_proj.weight] Loading weights: 97%|█████████▋| 303/311 [00:00<00:00, 406.45it/s, Materializing param=model.layers.27.mlp.up_proj.weight] Loading weights: 97%|█████████▋| 303/311 [00:00<00:00, 424.06it/s, Materializing param=model.layers.27.mlp.up_proj.weight] Loading weights: 89%|████████▉ | 277/311 [00:00<00:00, 369.38it/s, Materializing param=model.layers.24.self_attn.v_proj.weight] Loading weights: 91%|█████████ | 282/311 [00:00<00:00, 400.33it/s, Materializing param=model.layers.25.post_attention_layernorm.weight] Loading weights: 75%|███████▌ | 234/311 [00:00<00:00, 327.90it/s, Materializing param=model.layers.21.input_layernorm.weight] Loading weights: 75%|███████▌ | 234/311 [00:00<00:00, 327.90it/s, Materializing param=model.layers.21.input_layernorm.weight] Loading weights: 91%|█████████ | 282/311 [00:00<00:00, 400.33it/s, Materializing param=model.layers.25.post_attention_layernorm.weight] Loading weights: 76%|███████▌ | 235/311 [00:00<00:00, 327.90it/s, Materializing param=model.layers.21.mlp.down_proj.weight] Loading weights: 76%|███████▌ | 235/311 [00:00<00:00, 327.90it/s, Materializing param=model.layers.21.mlp.down_proj.weight] Loading weights: 91%|█████████ | 283/311 [00:00<00:00, 400.33it/s, Materializing param=model.layers.25.self_attn.k_norm.weight] Loading weights: 91%|█████████ | 283/311 [00:00<00:00, 400.33it/s, Materializing param=model.layers.25.self_attn.k_norm.weight] Loading weights: 91%|█████████▏| 284/311 [00:00<00:00, 400.33it/s, Materializing param=model.layers.25.self_attn.k_proj.weight] Loading weights: 91%|█████████▏| 284/311 [00:00<00:00, 400.33it/s, Materializing param=model.layers.25.self_attn.k_proj.weight] Loading weights: 92%|█████████▏| 285/311 [00:00<00:00, 400.33it/s, Materializing param=model.layers.25.self_attn.o_proj.weight] Loading weights: 92%|█████████▏| 285/311 [00:00<00:00, 400.33it/s, Materializing param=model.layers.25.self_attn.o_proj.weight] Loading weights: 89%|████████▉ | 278/311 [00:00<00:00, 369.38it/s, Materializing param=model.layers.25.input_layernorm.weight] Loading weights: 89%|████████▉ | 278/311 [00:00<00:00, 369.38it/s, Materializing param=model.layers.25.input_layernorm.weight] Loading weights: 90%|████████▉ | 279/311 [00:00<00:00, 369.38it/s, Materializing param=model.layers.25.mlp.down_proj.weight] Loading weights: 90%|████████▉ | 279/311 [00:00<00:00, 369.38it/s, Materializing param=model.layers.25.mlp.down_proj.weight] Loading weights: 69%|██████▉ | 215/311 [00:00<00:00, 315.03it/s, Materializing param=model.layers.19.mlp.up_proj.weight] Loading weights: 69%|██████▉ | 215/311 [00:00<00:00, 315.03it/s, Materializing param=model.layers.19.mlp.up_proj.weight] Loading weights: 92%|█████████▏| 286/311 [00:00<00:00, 400.33it/s, Materializing param=model.layers.25.self_attn.q_norm.weight] Loading weights: 92%|█████████▏| 286/311 [00:00<00:00, 400.33it/s, Materializing param=model.layers.25.self_attn.q_norm.weight] Loading weights: 92%|█████████▏| 287/311 [00:00<00:00, 400.33it/s, Materializing param=model.layers.25.self_attn.q_proj.weight] Loading weights: 92%|█████████▏| 287/311 [00:00<00:00, 400.33it/s, Materializing param=model.layers.25.self_attn.q_proj.weight] Loading weights: 62%|██████▏ | 193/311 [00:00<00:00, 306.36it/s, Materializing param=model.layers.17.mlp.up_proj.weight] Loading weights: 62%|██████▏ | 193/311 [00:00<00:00, 306.36it/s, Materializing param=model.layers.17.mlp.up_proj.weight] Loading weights: 93%|█████████▎| 288/311 [00:00<00:00, 400.33it/s, Materializing param=model.layers.25.self_attn.v_proj.weight] Loading weights: 93%|█████████▎| 288/311 [00:00<00:00, 400.33it/s, Materializing param=model.layers.25.self_attn.v_proj.weight] Loading weights: 93%|█████████▎| 289/311 [00:00<00:00, 400.33it/s, Materializing param=model.layers.26.input_layernorm.weight] Loading weights: 93%|█████████▎| 289/311 [00:00<00:00, 400.33it/s, Materializing param=model.layers.26.input_layernorm.weight] Loading weights: 93%|█████████▎| 290/311 [00:00<00:00, 400.33it/s, Materializing param=model.layers.26.mlp.down_proj.weight] Loading weights: 98%|█████████▊| 304/311 [00:00<00:00, 424.06it/s, Materializing param=model.layers.27.post_attention_layernorm.weight] Loading weights: 98%|█████████▊| 304/311 [00:00<00:00, 406.45it/s, Materializing param=model.layers.27.post_attention_layernorm.weight] Loading weights: 93%|█████████▎| 290/311 [00:00<00:00, 400.33it/s, Materializing param=model.layers.26.mlp.down_proj.weight] Loading weights: 98%|█████████▊| 304/311 [00:00<00:00, 406.45it/s, Materializing param=model.layers.27.post_attention_layernorm.weight] Loading weights: 98%|█████████▊| 304/311 [00:00<00:00, 424.06it/s, Materializing param=model.layers.27.post_attention_layernorm.weight] Loading weights: 98%|█████████▊| 305/311 [00:00<00:00, 406.45it/s, Materializing param=model.layers.27.self_attn.k_norm.weight] Loading weights: 98%|█████████▊| 305/311 [00:00<00:00, 406.45it/s, Materializing param=model.layers.27.self_attn.k_norm.weight] Loading weights: 98%|█████████▊| 306/311 [00:00<00:00, 406.45it/s, Materializing param=model.layers.27.self_attn.k_proj.weight] Loading weights: 98%|█████████▊| 306/311 [00:00<00:00, 406.45it/s, Materializing param=model.layers.27.self_attn.k_proj.weight] Loading weights: 98%|█████████▊| 305/311 [00:00<00:00, 424.06it/s, Materializing param=model.layers.27.self_attn.k_norm.weight] Loading weights: 98%|█████████▊| 305/311 [00:00<00:00, 424.06it/s, Materializing param=model.layers.27.self_attn.k_norm.weight] Loading weights: 98%|█████████▊| 306/311 [00:00<00:00, 424.06it/s, Materializing param=model.layers.27.self_attn.k_proj.weight] Loading weights: 98%|█████████▊| 306/311 [00:00<00:00, 424.06it/s, Materializing param=model.layers.27.self_attn.k_proj.weight] Loading weights: 99%|█████████▊| 307/311 [00:00<00:00, 406.45it/s, Materializing param=model.layers.27.self_attn.o_proj.weight] Loading weights: 99%|█████████▊| 307/311 [00:00<00:00, 406.45it/s, Materializing param=model.layers.27.self_attn.o_proj.weight] Loading weights: 99%|█████████▊| 307/311 [00:00<00:00, 424.06it/s, Materializing param=model.layers.27.self_attn.o_proj.weight] Loading weights: 99%|█████████▊| 307/311 [00:00<00:00, 424.06it/s, Materializing param=model.layers.27.self_attn.o_proj.weight] Loading weights: 90%|█████████ | 280/311 [00:00<00:00, 369.38it/s, Materializing param=model.layers.25.mlp.gate_proj.weight] Loading weights: 90%|█████████ | 280/311 [00:00<00:00, 369.38it/s, Materializing param=model.layers.25.mlp.gate_proj.weight] Loading weights: 99%|█████████▉| 308/311 [00:00<00:00, 406.45it/s, Materializing param=model.layers.27.self_attn.q_norm.weight] Loading weights: 99%|█████████▉| 308/311 [00:00<00:00, 424.06it/s, Materializing param=model.layers.27.self_attn.q_norm.weight] Loading weights: 99%|█████████▉| 308/311 [00:00<00:00, 406.45it/s, Materializing param=model.layers.27.self_attn.q_norm.weight] Loading weights: 99%|█████████▉| 308/311 [00:00<00:00, 424.06it/s, Materializing param=model.layers.27.self_attn.q_norm.weight] Loading weights: 69%|██████▉ | 216/311 [00:00<00:00, 315.03it/s, Materializing param=model.layers.19.post_attention_layernorm.weight] Loading weights: 69%|██████▉ | 216/311 [00:00<00:00, 315.03it/s, Materializing param=model.layers.19.post_attention_layernorm.weight] Loading weights: 70%|██████▉ | 217/311 [00:00<00:00, 315.03it/s, Materializing param=model.layers.19.self_attn.k_norm.weight] Loading weights: 70%|██████▉ | 217/311 [00:00<00:00, 315.03it/s, Materializing param=model.layers.19.self_attn.k_norm.weight] Loading weights: 70%|███████ | 218/311 [00:00<00:00, 315.03it/s, Materializing param=model.layers.19.self_attn.k_proj.weight] Loading weights: 70%|███████ | 218/311 [00:00<00:00, 315.03it/s, Materializing param=model.layers.19.self_attn.k_proj.weight] Loading weights: 70%|███████ | 219/311 [00:00<00:00, 315.03it/s, Materializing param=model.layers.19.self_attn.o_proj.weight] Loading weights: 70%|███████ | 219/311 [00:00<00:00, 315.03it/s, Materializing param=model.layers.19.self_attn.o_proj.weight] Loading weights: 62%|██████▏ | 194/311 [00:00<00:00, 306.36it/s, Materializing param=model.layers.17.post_attention_layernorm.weight] Loading weights: 62%|██████▏ | 194/311 [00:00<00:00, 306.36it/s, Materializing param=model.layers.17.post_attention_layernorm.weight] Loading weights: 99%|█████████▉| 309/311 [00:00<00:00, 424.06it/s, Materializing param=model.layers.27.self_attn.q_proj.weight] Loading weights: 99%|█████████▉| 309/311 [00:00<00:00, 424.06it/s, Materializing param=model.layers.27.self_attn.q_proj.weight] Loading weights: 99%|█████████▉| 309/311 [00:00<00:00, 406.45it/s, Materializing param=model.layers.27.self_attn.q_proj.weight] Loading weights: 99%|█████████▉| 309/311 [00:00<00:00, 406.45it/s, Materializing param=model.layers.27.self_attn.q_proj.weight] Loading weights: 94%|█████████▎| 291/311 [00:00<00:00, 400.33it/s, Materializing param=model.layers.26.mlp.gate_proj.weight] Loading weights: 63%|██████▎ | 195/311 [00:00<00:00, 306.36it/s, Materializing param=model.layers.17.self_attn.k_norm.weight] Loading weights: 100%|█████████▉| 310/311 [00:00<00:00, 424.06it/s, Materializing param=model.layers.27.self_attn.v_proj.weight] Loading weights: 63%|██████▎ | 195/311 [00:00<00:00, 306.36it/s, Materializing param=model.layers.17.self_attn.k_norm.weight] Loading weights: 100%|█████████▉| 310/311 [00:00<00:00, 424.06it/s, Materializing param=model.layers.27.self_attn.v_proj.weight] Loading weights: 94%|█████████▎| 291/311 [00:00<00:00, 400.33it/s, Materializing param=model.layers.26.mlp.gate_proj.weight] Loading weights: 100%|█████████▉| 310/311 [00:00<00:00, 406.45it/s, Materializing param=model.layers.27.self_attn.v_proj.weight] Loading weights: 100%|█████████▉| 310/311 [00:00<00:00, 406.45it/s, Materializing param=model.layers.27.self_attn.v_proj.weight] Loading weights: 63%|██████▎ | 196/311 [00:00<00:00, 306.36it/s, Materializing param=model.layers.17.self_attn.k_proj.weight] Loading weights: 63%|██████▎ | 196/311 [00:00<00:00, 306.36it/s, Materializing param=model.layers.17.self_attn.k_proj.weight] Loading weights: 63%|██████▎ | 197/311 [00:00<00:00, 306.36it/s, Materializing param=model.layers.17.self_attn.o_proj.weight] Loading weights: 100%|██████████| 311/311 [00:00<00:00, 406.45it/s, Materializing param=model.norm.weight] Loading weights: 63%|██████▎ | 197/311 [00:00<00:00, 306.36it/s, Materializing param=model.layers.17.self_attn.o_proj.weight] Loading weights: 100%|██████████| 311/311 [00:00<00:00, 424.06it/s, Materializing param=model.norm.weight] Loading weights: 100%|██████████| 311/311 [00:00<00:00, 406.45it/s, Materializing param=model.norm.weight] Loading weights: 100%|██████████| 311/311 [00:00<00:00, 424.06it/s, Materializing param=model.norm.weight] Loading weights: 100%|██████████| 311/311 [00:00<00:00, 322.60it/s, Materializing param=model.norm.weight] + Loading weights: 100%|██████████| 311/311 [00:00<00:00, 329.56it/s, Materializing param=model.norm.weight] + Loading weights: 71%|███████ | 220/311 [00:00<00:00, 315.03it/s, Materializing param=model.layers.19.self_attn.q_norm.weight] Loading weights: 71%|███████ | 220/311 [00:00<00:00, 315.03it/s, Materializing param=model.layers.19.self_attn.q_norm.weight] Loading weights: 71%|███████ | 221/311 [00:00<00:00, 315.03it/s, Materializing param=model.layers.19.self_attn.q_proj.weight] Loading weights: 71%|███████ | 221/311 [00:00<00:00, 315.03it/s, Materializing param=model.layers.19.self_attn.q_proj.weight] Loading weights: 90%|█████████ | 281/311 [00:00<00:00, 369.38it/s, Materializing param=model.layers.25.mlp.up_proj.weight] Loading weights: 76%|███████▌ | 236/311 [00:00<00:00, 327.90it/s, Materializing param=model.layers.21.mlp.gate_proj.weight] Loading weights: 90%|█████████ | 281/311 [00:00<00:00, 369.38it/s, Materializing param=model.layers.25.mlp.up_proj.weight] Loading weights: 76%|███████▌ | 236/311 [00:00<00:00, 327.90it/s, Materializing param=model.layers.21.mlp.gate_proj.weight] Loading weights: 76%|███████▌ | 237/311 [00:00<00:00, 327.90it/s, Materializing param=model.layers.21.mlp.up_proj.weight] Loading weights: 76%|███████▌ | 237/311 [00:00<00:00, 327.90it/s, Materializing param=model.layers.21.mlp.up_proj.weight] Loading weights: 64%|██████▎ | 198/311 [00:00<00:00, 329.16it/s, Materializing param=model.layers.17.self_attn.o_proj.weight] Loading weights: 64%|██████▎ | 198/311 [00:00<00:00, 329.16it/s, Materializing param=model.layers.17.self_attn.q_norm.weight] Loading weights: 64%|██████▎ | 198/311 [00:00<00:00, 329.16it/s, Materializing param=model.layers.17.self_attn.q_norm.weight] Loading weights: 64%|██████▍ | 199/311 [00:00<00:00, 329.16it/s, Materializing param=model.layers.17.self_attn.q_proj.weight] Loading weights: 64%|██████▍ | 199/311 [00:00<00:00, 329.16it/s, Materializing param=model.layers.17.self_attn.q_proj.weight] Loading weights: 71%|███████▏ | 222/311 [00:00<00:00, 315.03it/s, Materializing param=model.layers.19.self_attn.v_proj.weight] Loading weights: 71%|███████▏ | 222/311 [00:00<00:00, 315.03it/s, Materializing param=model.layers.19.self_attn.v_proj.weight] Loading weights: 72%|███████▏ | 223/311 [00:00<00:00, 315.03it/s, Materializing param=model.layers.20.input_layernorm.weight] Loading weights: 72%|███████▏ | 223/311 [00:00<00:00, 315.03it/s, Materializing param=model.layers.20.input_layernorm.weight] Loading weights: 94%|█████████▍| 292/311 [00:00<00:00, 400.33it/s, Materializing param=model.layers.26.mlp.up_proj.weight] Loading weights: 72%|███████▏ | 224/311 [00:00<00:00, 315.03it/s, Materializing param=model.layers.20.mlp.down_proj.weight] Loading weights: 94%|█████████▍| 292/311 [00:00<00:00, 400.33it/s, Materializing param=model.layers.26.mlp.up_proj.weight] Loading weights: 72%|███████▏ | 224/311 [00:00<00:00, 315.03it/s, Materializing param=model.layers.20.mlp.down_proj.weight] Loading weights: 64%|██████▍ | 200/311 [00:00<00:00, 329.16it/s, Materializing param=model.layers.17.self_attn.v_proj.weight] Loading weights: 64%|██████▍ | 200/311 [00:00<00:00, 329.16it/s, Materializing param=model.layers.17.self_attn.v_proj.weight] Loading weights: 65%|██████▍ | 201/311 [00:00<00:00, 329.16it/s, Materializing param=model.layers.18.input_layernorm.weight] Loading weights: 65%|██████▍ | 201/311 [00:00<00:00, 329.16it/s, Materializing param=model.layers.18.input_layernorm.weight] Loading weights: 65%|██████▍ | 202/311 [00:00<00:00, 329.16it/s, Materializing param=model.layers.18.mlp.down_proj.weight] Loading weights: 65%|██████▍ | 202/311 [00:00<00:00, 329.16it/s, Materializing param=model.layers.18.mlp.down_proj.weight] Loading weights: 77%|███████▋ | 238/311 [00:00<00:00, 327.90it/s, Materializing param=model.layers.21.post_attention_layernorm.weight] Loading weights: 77%|███████▋ | 238/311 [00:00<00:00, 327.90it/s, Materializing param=model.layers.21.post_attention_layernorm.weight] Loading weights: 77%|███████▋ | 239/311 [00:00<00:00, 327.90it/s, Materializing param=model.layers.21.self_attn.k_norm.weight] Loading weights: 77%|███████▋ | 239/311 [00:00<00:00, 327.90it/s, Materializing param=model.layers.21.self_attn.k_norm.weight] Loading weights: 77%|███████▋ | 240/311 [00:00<00:00, 327.90it/s, Materializing param=model.layers.21.self_attn.k_proj.weight] Loading weights: 77%|███████▋ | 240/311 [00:00<00:00, 327.90it/s, Materializing param=model.layers.21.self_attn.k_proj.weight] Loading weights: 91%|█████████ | 282/311 [00:00<00:00, 369.38it/s, Materializing param=model.layers.25.post_attention_layernorm.weight] Loading weights: 91%|█████████ | 282/311 [00:00<00:00, 369.38it/s, Materializing param=model.layers.25.post_attention_layernorm.weight] Loading weights: 91%|█████████ | 283/311 [00:00<00:00, 369.38it/s, Materializing param=model.layers.25.self_attn.k_norm.weight] Loading weights: 91%|█████████ | 283/311 [00:00<00:00, 369.38it/s, Materializing param=model.layers.25.self_attn.k_norm.weight] Loading weights: 91%|█████████▏| 284/311 [00:00<00:00, 369.38it/s, Materializing param=model.layers.25.self_attn.k_proj.weight] Loading weights: 91%|█████████▏| 284/311 [00:00<00:00, 369.38it/s, Materializing param=model.layers.25.self_attn.k_proj.weight] Loading weights: 92%|█████████▏| 285/311 [00:00<00:00, 369.38it/s, Materializing param=model.layers.25.self_attn.o_proj.weight] Loading weights: 92%|█████████▏| 285/311 [00:00<00:00, 369.38it/s, Materializing param=model.layers.25.self_attn.o_proj.weight] Loading weights: 94%|█████████▍| 293/311 [00:00<00:00, 400.33it/s, Materializing param=model.layers.26.post_attention_layernorm.weight] Loading weights: 94%|█████████▍| 293/311 [00:00<00:00, 400.33it/s, Materializing param=model.layers.26.post_attention_layernorm.weight] Loading weights: 95%|█████████▍| 294/311 [00:00<00:00, 400.33it/s, Materializing param=model.layers.26.self_attn.k_norm.weight] Loading weights: 95%|█████████▍| 294/311 [00:00<00:00, 400.33it/s, Materializing param=model.layers.26.self_attn.k_norm.weight] Loading weights: 95%|█████████▍| 295/311 [00:00<00:00, 400.33it/s, Materializing param=model.layers.26.self_attn.k_proj.weight] Loading weights: 95%|█████████▍| 295/311 [00:00<00:00, 400.33it/s, Materializing param=model.layers.26.self_attn.k_proj.weight] Loading weights: 77%|███████▋ | 241/311 [00:00<00:00, 327.90it/s, Materializing param=model.layers.21.self_attn.o_proj.weight] Loading weights: 77%|███████▋ | 241/311 [00:00<00:00, 327.90it/s, Materializing param=model.layers.21.self_attn.o_proj.weight] Loading weights: 72%|███████▏ | 225/311 [00:00<00:00, 315.03it/s, Materializing param=model.layers.20.mlp.gate_proj.weight] Loading weights: 78%|███████▊ | 242/311 [00:00<00:00, 327.90it/s, Materializing param=model.layers.21.self_attn.q_norm.weight] Loading weights: 78%|███████▊ | 242/311 [00:00<00:00, 327.90it/s, Materializing param=model.layers.21.self_attn.q_norm.weight] Loading weights: 95%|█████████▌| 296/311 [00:00<00:00, 400.33it/s, Materializing param=model.layers.26.self_attn.o_proj.weight] Loading weights: 72%|███████▏ | 225/311 [00:00<00:00, 315.03it/s, Materializing param=model.layers.20.mlp.gate_proj.weight] Loading weights: 95%|█████████▌| 296/311 [00:00<00:00, 400.33it/s, Materializing param=model.layers.26.self_attn.o_proj.weight] Loading weights: 92%|█████████▏| 286/311 [00:00<00:00, 369.38it/s, Materializing param=model.layers.25.self_attn.q_norm.weight] Loading weights: 92%|█████████▏| 286/311 [00:00<00:00, 369.38it/s, Materializing param=model.layers.25.self_attn.q_norm.weight] Loading weights: 92%|█████████▏| 287/311 [00:00<00:00, 369.38it/s, Materializing param=model.layers.25.self_attn.q_proj.weight] Loading weights: 92%|█████████▏| 287/311 [00:00<00:00, 369.38it/s, Materializing param=model.layers.25.self_attn.q_proj.weight] Loading weights: 65%|██████▌ | 203/311 [00:00<00:00, 329.16it/s, Materializing param=model.layers.18.mlp.gate_proj.weight] Loading weights: 65%|██████▌ | 203/311 [00:00<00:00, 329.16it/s, Materializing param=model.layers.18.mlp.gate_proj.weight] Loading weights: 78%|███████▊ | 243/311 [00:00<00:00, 327.90it/s, Materializing param=model.layers.21.self_attn.q_proj.weight] Loading weights: 78%|███████▊ | 243/311 [00:00<00:00, 327.90it/s, Materializing param=model.layers.21.self_attn.q_proj.weight] Loading weights: 78%|███████▊ | 244/311 [00:00<00:00, 327.90it/s, Materializing param=model.layers.21.self_attn.v_proj.weight] Loading weights: 78%|███████▊ | 244/311 [00:00<00:00, 327.90it/s, Materializing param=model.layers.21.self_attn.v_proj.weight] Loading weights: 95%|█████████▌| 297/311 [00:00<00:00, 400.33it/s, Materializing param=model.layers.26.self_attn.q_norm.weight] Loading weights: 95%|█████████▌| 297/311 [00:00<00:00, 400.33it/s, Materializing param=model.layers.26.self_attn.q_norm.weight] Loading weights: 96%|█████████▌| 298/311 [00:00<00:00, 400.33it/s, Materializing param=model.layers.26.self_attn.q_proj.weight] Loading weights: 96%|█████████▌| 298/311 [00:00<00:00, 400.33it/s, Materializing param=model.layers.26.self_attn.q_proj.weight] Loading weights: 79%|███████▉ | 245/311 [00:00<00:00, 327.90it/s, Materializing param=model.layers.22.input_layernorm.weight] Loading weights: 79%|███████▉ | 245/311 [00:00<00:00, 327.90it/s, Materializing param=model.layers.22.input_layernorm.weight] Loading weights: 79%|███████▉ | 246/311 [00:00<00:00, 327.90it/s, Materializing param=model.layers.22.mlp.down_proj.weight] Loading weights: 79%|███████▉ | 246/311 [00:00<00:00, 327.90it/s, Materializing param=model.layers.22.mlp.down_proj.weight] Loading weights: 93%|█████████▎| 288/311 [00:00<00:00, 369.38it/s, Materializing param=model.layers.25.self_attn.v_proj.weight] Loading weights: 93%|█████████▎| 288/311 [00:00<00:00, 369.38it/s, Materializing param=model.layers.25.self_attn.v_proj.weight] Loading weights: 93%|█████████▎| 289/311 [00:00<00:00, 369.38it/s, Materializing param=model.layers.26.input_layernorm.weight] Loading weights: 93%|█████████▎| 289/311 [00:00<00:00, 369.38it/s, Materializing param=model.layers.26.input_layernorm.weight] Loading weights: 96%|█████████▌| 299/311 [00:00<00:00, 400.33it/s, Materializing param=model.layers.26.self_attn.v_proj.weight] Loading weights: 96%|█████████▌| 299/311 [00:00<00:00, 400.33it/s, Materializing param=model.layers.26.self_attn.v_proj.weight] Loading weights: 93%|█████████▎| 290/311 [00:00<00:00, 369.38it/s, Materializing param=model.layers.26.mlp.down_proj.weight] Loading weights: 93%|█████████▎| 290/311 [00:00<00:00, 369.38it/s, Materializing param=model.layers.26.mlp.down_proj.weight] Loading weights: 96%|█████████▋| 300/311 [00:00<00:00, 400.33it/s, Materializing param=model.layers.27.input_layernorm.weight] Loading weights: 96%|█████████▋| 300/311 [00:00<00:00, 400.33it/s, Materializing param=model.layers.27.input_layernorm.weight] Loading weights: 97%|█████████▋| 301/311 [00:00<00:00, 400.33it/s, Materializing param=model.layers.27.mlp.down_proj.weight] Loading weights: 97%|█████████▋| 301/311 [00:00<00:00, 400.33it/s, Materializing param=model.layers.27.mlp.down_proj.weight] Loading weights: 73%|███████▎ | 226/311 [00:00<00:00, 315.03it/s, Materializing param=model.layers.20.mlp.up_proj.weight] Loading weights: 73%|███████▎ | 226/311 [00:00<00:00, 315.03it/s, Materializing param=model.layers.20.mlp.up_proj.weight] Loading weights: 66%|██████▌ | 204/311 [00:00<00:00, 329.16it/s, Materializing param=model.layers.18.mlp.up_proj.weight] Loading weights: 66%|██████▌ | 204/311 [00:00<00:00, 329.16it/s, Materializing param=model.layers.18.mlp.up_proj.weight] Loading weights: 97%|█████████▋| 302/311 [00:00<00:00, 400.33it/s, Materializing param=model.layers.27.mlp.gate_proj.weight] Loading weights: 97%|█████████▋| 302/311 [00:00<00:00, 400.33it/s, Materializing param=model.layers.27.mlp.gate_proj.weight] Loading weights: 94%|█████████▎| 291/311 [00:00<00:00, 369.38it/s, Materializing param=model.layers.26.mlp.gate_proj.weight] Loading weights: 94%|█████████▎| 291/311 [00:00<00:00, 369.38it/s, Materializing param=model.layers.26.mlp.gate_proj.weight] Loading weights: 73%|███████▎ | 227/311 [00:00<00:00, 315.03it/s, Materializing param=model.layers.20.post_attention_layernorm.weight] Loading weights: 73%|███████▎ | 227/311 [00:00<00:00, 315.03it/s, Materializing param=model.layers.20.post_attention_layernorm.weight] Loading weights: 73%|███████▎ | 228/311 [00:00<00:00, 315.03it/s, Materializing param=model.layers.20.self_attn.k_norm.weight] Loading weights: 73%|███████▎ | 228/311 [00:00<00:00, 315.03it/s, Materializing param=model.layers.20.self_attn.k_norm.weight] Loading weights: 74%|███████▎ | 229/311 [00:00<00:00, 315.03it/s, Materializing param=model.layers.20.self_attn.k_proj.weight] Loading weights: 74%|███████▎ | 229/311 [00:00<00:00, 315.03it/s, Materializing param=model.layers.20.self_attn.k_proj.weight] Loading weights: 74%|███████▍ | 230/311 [00:00<00:00, 315.03it/s, Materializing param=model.layers.20.self_attn.o_proj.weight] Loading weights: 74%|███████▍ | 230/311 [00:00<00:00, 315.03it/s, Materializing param=model.layers.20.self_attn.o_proj.weight] Loading weights: 79%|███████▉ | 247/311 [00:00<00:00, 327.90it/s, Materializing param=model.layers.22.mlp.gate_proj.weight] Loading weights: 79%|███████▉ | 247/311 [00:00<00:00, 327.90it/s, Materializing param=model.layers.22.mlp.gate_proj.weight] Loading weights: 66%|██████▌ | 205/311 [00:00<00:00, 329.16it/s, Materializing param=model.layers.18.post_attention_layernorm.weight] Loading weights: 80%|███████▉ | 248/311 [00:00<00:00, 327.90it/s, Materializing param=model.layers.22.mlp.up_proj.weight] Loading weights: 80%|███████▉ | 248/311 [00:00<00:00, 327.90it/s, Materializing param=model.layers.22.mlp.up_proj.weight] Loading weights: 66%|██████▌ | 205/311 [00:00<00:00, 329.16it/s, Materializing param=model.layers.18.post_attention_layernorm.weight] Loading weights: 66%|██████▌ | 206/311 [00:00<00:00, 329.16it/s, Materializing param=model.layers.18.self_attn.k_norm.weight] Loading weights: 66%|██████▌ | 206/311 [00:00<00:00, 329.16it/s, Materializing param=model.layers.18.self_attn.k_norm.weight] Loading weights: 67%|██████▋ | 207/311 [00:00<00:00, 329.16it/s, Materializing param=model.layers.18.self_attn.k_proj.weight] Loading weights: 67%|██████▋ | 207/311 [00:00<00:00, 329.16it/s, Materializing param=model.layers.18.self_attn.k_proj.weight] Loading weights: 67%|██████▋ | 208/311 [00:00<00:00, 329.16it/s, Materializing param=model.layers.18.self_attn.o_proj.weight] Loading weights: 67%|██████▋ | 208/311 [00:00<00:00, 329.16it/s, Materializing param=model.layers.18.self_attn.o_proj.weight] Loading weights: 97%|█████████▋| 303/311 [00:00<00:00, 400.33it/s, Materializing param=model.layers.27.mlp.up_proj.weight] Loading weights: 74%|███████▍ | 231/311 [00:00<00:00, 334.15it/s, Materializing param=model.layers.20.self_attn.o_proj.weight] Loading weights: 97%|█████████▋| 303/311 [00:00<00:00, 400.33it/s, Materializing param=model.layers.27.mlp.up_proj.weight] Loading weights: 74%|███████▍ | 231/311 [00:00<00:00, 334.15it/s, Materializing param=model.layers.20.self_attn.q_norm.weight] Loading weights: 74%|███████▍ | 231/311 [00:00<00:00, 334.15it/s, Materializing param=model.layers.20.self_attn.q_norm.weight] Loading weights: 94%|█████████▍| 292/311 [00:00<00:00, 369.38it/s, Materializing param=model.layers.26.mlp.up_proj.weight] Loading weights: 94%|█████████▍| 292/311 [00:00<00:00, 369.38it/s, Materializing param=model.layers.26.mlp.up_proj.weight] Loading weights: 67%|██████▋ | 209/311 [00:00<00:00, 329.16it/s, Materializing param=model.layers.18.self_attn.q_norm.weight] Loading weights: 67%|██████▋ | 209/311 [00:00<00:00, 329.16it/s, Materializing param=model.layers.18.self_attn.q_norm.weight] Loading weights: 68%|██████▊ | 210/311 [00:00<00:00, 329.16it/s, Materializing param=model.layers.18.self_attn.q_proj.weight] Loading weights: 68%|██████▊ | 210/311 [00:00<00:00, 329.16it/s, Materializing param=model.layers.18.self_attn.q_proj.weight] Loading weights: 75%|███████▍ | 232/311 [00:00<00:00, 334.15it/s, Materializing param=model.layers.20.self_attn.q_proj.weight] Loading weights: 75%|███████▍ | 232/311 [00:00<00:00, 334.15it/s, Materializing param=model.layers.20.self_attn.q_proj.weight] Loading weights: 75%|███████▍ | 233/311 [00:00<00:00, 334.15it/s, Materializing param=model.layers.20.self_attn.v_proj.weight] Loading weights: 75%|███████▍ | 233/311 [00:00<00:00, 334.15it/s, Materializing param=model.layers.20.self_attn.v_proj.weight] Loading weights: 75%|███████▌ | 234/311 [00:00<00:00, 334.15it/s, Materializing param=model.layers.21.input_layernorm.weight] Loading weights: 75%|███████▌ | 234/311 [00:00<00:00, 334.15it/s, Materializing param=model.layers.21.input_layernorm.weight] Loading weights: 76%|███████▌ | 235/311 [00:00<00:00, 334.15it/s, Materializing param=model.layers.21.mlp.down_proj.weight] Loading weights: 76%|███████▌ | 235/311 [00:00<00:00, 334.15it/s, Materializing param=model.layers.21.mlp.down_proj.weight] Loading weights: 68%|██████▊ | 211/311 [00:00<00:00, 329.16it/s, Materializing param=model.layers.18.self_attn.v_proj.weight] Loading weights: 68%|██████▊ | 211/311 [00:00<00:00, 329.16it/s, Materializing param=model.layers.18.self_attn.v_proj.weight] Loading weights: 68%|██████▊ | 212/311 [00:00<00:00, 329.16it/s, Materializing param=model.layers.19.input_layernorm.weight] Loading weights: 68%|██████▊ | 212/311 [00:00<00:00, 329.16it/s, Materializing param=model.layers.19.input_layernorm.weight] Loading weights: 68%|██████▊ | 213/311 [00:00<00:00, 329.16it/s, Materializing param=model.layers.19.mlp.down_proj.weight] Loading weights: 80%|████████ | 249/311 [00:01<00:00, 327.90it/s, Materializing param=model.layers.22.post_attention_layernorm.weight] Loading weights: 68%|██████▊ | 213/311 [00:00<00:00, 329.16it/s, Materializing param=model.layers.19.mlp.down_proj.weight] Loading weights: 80%|████████ | 249/311 [00:01<00:00, 327.90it/s, Materializing param=model.layers.22.post_attention_layernorm.weight] Loading weights: 80%|████████ | 250/311 [00:01<00:00, 327.90it/s, Materializing param=model.layers.22.self_attn.k_norm.weight] Loading weights: 80%|████████ | 250/311 [00:01<00:00, 327.90it/s, Materializing param=model.layers.22.self_attn.k_norm.weight] Loading weights: 98%|█████████▊| 304/311 [00:00<00:00, 400.33it/s, Materializing param=model.layers.27.post_attention_layernorm.weight] Loading weights: 98%|█████████▊| 304/311 [00:00<00:00, 400.33it/s, Materializing param=model.layers.27.post_attention_layernorm.weight] Loading weights: 81%|████████ | 251/311 [00:01<00:00, 327.90it/s, Materializing param=model.layers.22.self_attn.k_proj.weight] Loading weights: 81%|████████ | 251/311 [00:01<00:00, 327.90it/s, Materializing param=model.layers.22.self_attn.k_proj.weight] Loading weights: 98%|█████████▊| 305/311 [00:00<00:00, 400.33it/s, Materializing param=model.layers.27.self_attn.k_norm.weight] Loading weights: 98%|█████████▊| 305/311 [00:00<00:00, 400.33it/s, Materializing param=model.layers.27.self_attn.k_norm.weight] Loading weights: 81%|████████ | 252/311 [00:01<00:00, 327.90it/s, Materializing param=model.layers.22.self_attn.o_proj.weight] Loading weights: 81%|████████ | 252/311 [00:01<00:00, 327.90it/s, Materializing param=model.layers.22.self_attn.o_proj.weight] Loading weights: 98%|█████████▊| 306/311 [00:00<00:00, 400.33it/s, Materializing param=model.layers.27.self_attn.k_proj.weight] Loading weights: 98%|█████████▊| 306/311 [00:00<00:00, 400.33it/s, Materializing param=model.layers.27.self_attn.k_proj.weight] Loading weights: 99%|█████████▊| 307/311 [00:00<00:00, 400.33it/s, Materializing param=model.layers.27.self_attn.o_proj.weight] Loading weights: 99%|█████████▊| 307/311 [00:00<00:00, 400.33it/s, Materializing param=model.layers.27.self_attn.o_proj.weight] Loading weights: 94%|█████████▍| 293/311 [00:00<00:00, 369.38it/s, Materializing param=model.layers.26.post_attention_layernorm.weight] Loading weights: 94%|█████████▍| 293/311 [00:00<00:00, 369.38it/s, Materializing param=model.layers.26.post_attention_layernorm.weight] Loading weights: 95%|█████████▍| 294/311 [00:01<00:00, 369.38it/s, Materializing param=model.layers.26.self_attn.k_norm.weight] Loading weights: 81%|████████▏ | 253/311 [00:01<00:00, 327.90it/s, Materializing param=model.layers.22.self_attn.q_norm.weight] Loading weights: 95%|█████████▍| 294/311 [00:01<00:00, 369.38it/s, Materializing param=model.layers.26.self_attn.k_norm.weight] Loading weights: 81%|████████▏ | 253/311 [00:01<00:00, 327.90it/s, Materializing param=model.layers.22.self_attn.q_norm.weight] Loading weights: 95%|█████████▍| 295/311 [00:01<00:00, 369.38it/s, Materializing param=model.layers.26.self_attn.k_proj.weight] Loading weights: 82%|████████▏ | 254/311 [00:01<00:00, 327.90it/s, Materializing param=model.layers.22.self_attn.q_proj.weight] Loading weights: 82%|████████▏ | 254/311 [00:01<00:00, 327.90it/s, Materializing param=model.layers.22.self_attn.q_proj.weight] Loading weights: 95%|█████████▍| 295/311 [00:01<00:00, 369.38it/s, Materializing param=model.layers.26.self_attn.k_proj.weight] Loading weights: 99%|█████████▉| 308/311 [00:00<00:00, 400.33it/s, Materializing param=model.layers.27.self_attn.q_norm.weight] Loading weights: 99%|█████████▉| 308/311 [00:00<00:00, 400.33it/s, Materializing param=model.layers.27.self_attn.q_norm.weight] Loading weights: 95%|█████████▌| 296/311 [00:01<00:00, 369.38it/s, Materializing param=model.layers.26.self_attn.o_proj.weight] Loading weights: 95%|█████████▌| 296/311 [00:01<00:00, 369.38it/s, Materializing param=model.layers.26.self_attn.o_proj.weight] Loading weights: 99%|█████████▉| 309/311 [00:00<00:00, 400.33it/s, Materializing param=model.layers.27.self_attn.q_proj.weight] Loading weights: 99%|█████████▉| 309/311 [00:00<00:00, 400.33it/s, Materializing param=model.layers.27.self_attn.q_proj.weight] Loading weights: 76%|███████▌ | 236/311 [00:00<00:00, 334.15it/s, Materializing param=model.layers.21.mlp.gate_proj.weight] Loading weights: 69%|██████▉ | 214/311 [00:00<00:00, 329.16it/s, Materializing param=model.layers.19.mlp.gate_proj.weight] Loading weights: 69%|██████▉ | 214/311 [00:00<00:00, 329.16it/s, Materializing param=model.layers.19.mlp.gate_proj.weight] Loading weights: 76%|███████▌ | 236/311 [00:00<00:00, 334.15it/s, Materializing param=model.layers.21.mlp.gate_proj.weight] Loading weights: 100%|█████████▉| 310/311 [00:00<00:00, 400.33it/s, Materializing param=model.layers.27.self_attn.v_proj.weight] Loading weights: 100%|█████████▉| 310/311 [00:00<00:00, 400.33it/s, Materializing param=model.layers.27.self_attn.v_proj.weight] Loading weights: 82%|████████▏ | 255/311 [00:01<00:00, 327.90it/s, Materializing param=model.layers.22.self_attn.v_proj.weight] Loading weights: 82%|████████▏ | 255/311 [00:01<00:00, 327.90it/s, Materializing param=model.layers.22.self_attn.v_proj.weight] Loading weights: 95%|█████████▌| 297/311 [00:01<00:00, 369.38it/s, Materializing param=model.layers.26.self_attn.q_norm.weight] Loading weights: 95%|█████████▌| 297/311 [00:01<00:00, 369.38it/s, Materializing param=model.layers.26.self_attn.q_norm.weight] Loading weights: 100%|██████████| 311/311 [00:00<00:00, 400.33it/s, Materializing param=model.norm.weight] Loading weights: 100%|██████████| 311/311 [00:00<00:00, 400.33it/s, Materializing param=model.norm.weight] Loading weights: 82%|████████▏ | 256/311 [00:01<00:00, 327.90it/s, Materializing param=model.layers.23.input_layernorm.weight] Loading weights: 100%|██████████| 311/311 [00:00<00:00, 323.93it/s, Materializing param=model.norm.weight] + Loading weights: 82%|████████▏ | 256/311 [00:01<00:00, 327.90it/s, Materializing param=model.layers.23.input_layernorm.weight] Loading weights: 83%|████████▎ | 257/311 [00:01<00:00, 327.90it/s, Materializing param=model.layers.23.mlp.down_proj.weight] Loading weights: 83%|████████▎ | 257/311 [00:01<00:00, 327.90it/s, Materializing param=model.layers.23.mlp.down_proj.weight] Loading weights: 96%|█████████▌| 298/311 [00:01<00:00, 369.38it/s, Materializing param=model.layers.26.self_attn.q_proj.weight] Loading weights: 96%|█████████▌| 298/311 [00:01<00:00, 369.38it/s, Materializing param=model.layers.26.self_attn.q_proj.weight] Loading weights: 96%|█████████▌| 299/311 [00:01<00:00, 369.38it/s, Materializing param=model.layers.26.self_attn.v_proj.weight] Loading weights: 96%|█████████▌| 299/311 [00:01<00:00, 369.38it/s, Materializing param=model.layers.26.self_attn.v_proj.weight] Loading weights: 96%|█████████▋| 300/311 [00:01<00:00, 369.38it/s, Materializing param=model.layers.27.input_layernorm.weight] Loading weights: 96%|█████████▋| 300/311 [00:01<00:00, 369.38it/s, Materializing param=model.layers.27.input_layernorm.weight] Loading weights: 97%|█████████▋| 301/311 [00:01<00:00, 369.38it/s, Materializing param=model.layers.27.mlp.down_proj.weight] Loading weights: 97%|█████████▋| 301/311 [00:01<00:00, 369.38it/s, Materializing param=model.layers.27.mlp.down_proj.weight] Loading weights: 69%|██████▉ | 215/311 [00:00<00:00, 329.16it/s, Materializing param=model.layers.19.mlp.up_proj.weight] Loading weights: 76%|███████▌ | 237/311 [00:00<00:00, 334.15it/s, Materializing param=model.layers.21.mlp.up_proj.weight] Loading weights: 76%|███████▌ | 237/311 [00:00<00:00, 334.15it/s, Materializing param=model.layers.21.mlp.up_proj.weight] Loading weights: 69%|██████▉ | 215/311 [00:00<00:00, 329.16it/s, Materializing param=model.layers.19.mlp.up_proj.weight] Loading weights: 83%|████████▎ | 258/311 [00:01<00:00, 327.90it/s, Materializing param=model.layers.23.mlp.gate_proj.weight] Loading weights: 83%|████████▎ | 258/311 [00:01<00:00, 327.90it/s, Materializing param=model.layers.23.mlp.gate_proj.weight] Loading weights: 97%|█████████▋| 302/311 [00:01<00:00, 383.44it/s, Materializing param=model.layers.27.mlp.down_proj.weight] Loading weights: 97%|█████████▋| 302/311 [00:01<00:00, 383.44it/s, Materializing param=model.layers.27.mlp.gate_proj.weight] Loading weights: 97%|█████████▋| 302/311 [00:01<00:00, 383.44it/s, Materializing param=model.layers.27.mlp.gate_proj.weight] Loading weights: 77%|███████▋ | 238/311 [00:00<00:00, 334.15it/s, Materializing param=model.layers.21.post_attention_layernorm.weight] Loading weights: 69%|██████▉ | 216/311 [00:00<00:00, 329.16it/s, Materializing param=model.layers.19.post_attention_layernorm.weight] Loading weights: 77%|███████▋ | 238/311 [00:00<00:00, 334.15it/s, Materializing param=model.layers.21.post_attention_layernorm.weight] Loading weights: 77%|███████▋ | 239/311 [00:00<00:00, 334.15it/s, Materializing param=model.layers.21.self_attn.k_norm.weight] Loading weights: 77%|███████▋ | 239/311 [00:00<00:00, 334.15it/s, Materializing param=model.layers.21.self_attn.k_norm.weight] Loading weights: 69%|██████▉ | 216/311 [00:00<00:00, 329.16it/s, Materializing param=model.layers.19.post_attention_layernorm.weight] Loading weights: 77%|███████▋ | 240/311 [00:00<00:00, 334.15it/s, Materializing param=model.layers.21.self_attn.k_proj.weight] Loading weights: 77%|███████▋ | 240/311 [00:00<00:00, 334.15it/s, Materializing param=model.layers.21.self_attn.k_proj.weight] Loading weights: 70%|██████▉ | 217/311 [00:00<00:00, 329.16it/s, Materializing param=model.layers.19.self_attn.k_norm.weight] Loading weights: 70%|██████▉ | 217/311 [00:00<00:00, 329.16it/s, Materializing param=model.layers.19.self_attn.k_norm.weight] Loading weights: 77%|███████▋ | 241/311 [00:00<00:00, 334.15it/s, Materializing param=model.layers.21.self_attn.o_proj.weight] Loading weights: 70%|███████ | 218/311 [00:00<00:00, 329.16it/s, Materializing param=model.layers.19.self_attn.k_proj.weight] Loading weights: 77%|███████▋ | 241/311 [00:00<00:00, 334.15it/s, Materializing param=model.layers.21.self_attn.o_proj.weight] Loading weights: 83%|████████▎ | 259/311 [00:01<00:00, 327.90it/s, Materializing param=model.layers.23.mlp.up_proj.weight] Loading weights: 70%|███████ | 218/311 [00:00<00:00, 329.16it/s, Materializing param=model.layers.19.self_attn.k_proj.weight] Loading weights: 83%|████████▎ | 259/311 [00:01<00:00, 327.90it/s, Materializing param=model.layers.23.mlp.up_proj.weight] Loading weights: 70%|███████ | 219/311 [00:00<00:00, 329.16it/s, Materializing param=model.layers.19.self_attn.o_proj.weight] Loading weights: 70%|███████ | 219/311 [00:00<00:00, 329.16it/s, Materializing param=model.layers.19.self_attn.o_proj.weight] Loading weights: 78%|███████▊ | 242/311 [00:00<00:00, 334.15it/s, Materializing param=model.layers.21.self_attn.q_norm.weight] Loading weights: 78%|███████▊ | 242/311 [00:00<00:00, 334.15it/s, Materializing param=model.layers.21.self_attn.q_norm.weight] Loading weights: 71%|███████ | 220/311 [00:00<00:00, 329.16it/s, Materializing param=model.layers.19.self_attn.q_norm.weight] Loading weights: 71%|███████ | 220/311 [00:00<00:00, 329.16it/s, Materializing param=model.layers.19.self_attn.q_norm.weight] Loading weights: 97%|█████████▋| 303/311 [00:01<00:00, 383.44it/s, Materializing param=model.layers.27.mlp.up_proj.weight] Loading weights: 97%|█████████▋| 303/311 [00:01<00:00, 383.44it/s, Materializing param=model.layers.27.mlp.up_proj.weight] Loading weights: 71%|███████ | 221/311 [00:00<00:00, 329.16it/s, Materializing param=model.layers.19.self_attn.q_proj.weight] Loading weights: 71%|███████ | 221/311 [00:00<00:00, 329.16it/s, Materializing param=model.layers.19.self_attn.q_proj.weight] Loading weights: 78%|███████▊ | 243/311 [00:00<00:00, 334.15it/s, Materializing param=model.layers.21.self_attn.q_proj.weight] Loading weights: 78%|███████▊ | 243/311 [00:00<00:00, 334.15it/s, Materializing param=model.layers.21.self_attn.q_proj.weight] Loading weights: 71%|███████▏ | 222/311 [00:00<00:00, 329.16it/s, Materializing param=model.layers.19.self_attn.v_proj.weight] Loading weights: 71%|███████▏ | 222/311 [00:00<00:00, 329.16it/s, Materializing param=model.layers.19.self_attn.v_proj.weight] Loading weights: 78%|███████▊ | 244/311 [00:00<00:00, 334.15it/s, Materializing param=model.layers.21.self_attn.v_proj.weight] Loading weights: 78%|███████▊ | 244/311 [00:00<00:00, 334.15it/s, Materializing param=model.layers.21.self_attn.v_proj.weight] Loading weights: 72%|███████▏ | 223/311 [00:00<00:00, 329.16it/s, Materializing param=model.layers.20.input_layernorm.weight] Loading weights: 72%|███████▏ | 223/311 [00:00<00:00, 329.16it/s, Materializing param=model.layers.20.input_layernorm.weight] Loading weights: 79%|███████▉ | 245/311 [00:00<00:00, 334.15it/s, Materializing param=model.layers.22.input_layernorm.weight] Loading weights: 72%|███████▏ | 224/311 [00:00<00:00, 329.16it/s, Materializing param=model.layers.20.mlp.down_proj.weight] Loading weights: 79%|███████▉ | 245/311 [00:00<00:00, 334.15it/s, Materializing param=model.layers.22.input_layernorm.weight] Loading weights: 72%|███████▏ | 224/311 [00:00<00:00, 329.16it/s, Materializing param=model.layers.20.mlp.down_proj.weight] Loading weights: 79%|███████▉ | 246/311 [00:00<00:00, 334.15it/s, Materializing param=model.layers.22.mlp.down_proj.weight] Loading weights: 79%|███████▉ | 246/311 [00:00<00:00, 334.15it/s, Materializing param=model.layers.22.mlp.down_proj.weight] Loading weights: 84%|████████▎ | 260/311 [00:01<00:00, 327.90it/s, Materializing param=model.layers.23.post_attention_layernorm.weight] Loading weights: 84%|████████▎ | 260/311 [00:01<00:00, 327.90it/s, Materializing param=model.layers.23.post_attention_layernorm.weight] Loading weights: 84%|████████▍ | 261/311 [00:01<00:00, 327.90it/s, Materializing param=model.layers.23.self_attn.k_norm.weight] Loading weights: 84%|████████▍ | 261/311 [00:01<00:00, 327.90it/s, Materializing param=model.layers.23.self_attn.k_norm.weight] Loading weights: 84%|████████▍ | 262/311 [00:01<00:00, 327.90it/s, Materializing param=model.layers.23.self_attn.k_proj.weight] Loading weights: 84%|████████▍ | 262/311 [00:01<00:00, 327.90it/s, Materializing param=model.layers.23.self_attn.k_proj.weight] Loading weights: 85%|████████▍ | 263/311 [00:01<00:00, 327.90it/s, Materializing param=model.layers.23.self_attn.o_proj.weight] Loading weights: 85%|████████▍ | 263/311 [00:01<00:00, 327.90it/s, Materializing param=model.layers.23.self_attn.o_proj.weight] Loading weights: 85%|████████▍ | 264/311 [00:01<00:00, 327.90it/s, Materializing param=model.layers.23.self_attn.q_norm.weight] Loading weights: 85%|████████▍ | 264/311 [00:01<00:00, 327.90it/s, Materializing param=model.layers.23.self_attn.q_norm.weight] Loading weights: 98%|█████████▊| 304/311 [00:01<00:00, 383.44it/s, Materializing param=model.layers.27.post_attention_layernorm.weight] Loading weights: 98%|█████████▊| 304/311 [00:01<00:00, 383.44it/s, Materializing param=model.layers.27.post_attention_layernorm.weight] Loading weights: 98%|█████████▊| 305/311 [00:01<00:00, 383.44it/s, Materializing param=model.layers.27.self_attn.k_norm.weight] Loading weights: 98%|█████████▊| 305/311 [00:01<00:00, 383.44it/s, Materializing param=model.layers.27.self_attn.k_norm.weight] Loading weights: 98%|█████████▊| 306/311 [00:01<00:00, 383.44it/s, Materializing param=model.layers.27.self_attn.k_proj.weight] Loading weights: 98%|█████████▊| 306/311 [00:01<00:00, 383.44it/s, Materializing param=model.layers.27.self_attn.k_proj.weight] Loading weights: 99%|█████████▊| 307/311 [00:01<00:00, 383.44it/s, Materializing param=model.layers.27.self_attn.o_proj.weight] Loading weights: 99%|█████████▊| 307/311 [00:01<00:00, 383.44it/s, Materializing param=model.layers.27.self_attn.o_proj.weight] Loading weights: 72%|███████▏ | 225/311 [00:00<00:00, 329.16it/s, Materializing param=model.layers.20.mlp.gate_proj.weight] Loading weights: 72%|███████▏ | 225/311 [00:00<00:00, 329.16it/s, Materializing param=model.layers.20.mlp.gate_proj.weight] Loading weights: 79%|███████▉ | 247/311 [00:00<00:00, 334.15it/s, Materializing param=model.layers.22.mlp.gate_proj.weight] Loading weights: 85%|████████▌ | 265/311 [00:01<00:00, 327.90it/s, Materializing param=model.layers.23.self_attn.q_proj.weight] Loading weights: 79%|███████▉ | 247/311 [00:00<00:00, 334.15it/s, Materializing param=model.layers.22.mlp.gate_proj.weight] Loading weights: 85%|████████▌ | 265/311 [00:01<00:00, 327.90it/s, Materializing param=model.layers.23.self_attn.q_proj.weight] Loading weights: 86%|████████▌ | 266/311 [00:01<00:00, 327.90it/s, Materializing param=model.layers.23.self_attn.v_proj.weight] Loading weights: 86%|████████▌ | 266/311 [00:01<00:00, 327.90it/s, Materializing param=model.layers.23.self_attn.v_proj.weight] Loading weights: 86%|████████▌ | 267/311 [00:01<00:00, 327.90it/s, Materializing param=model.layers.24.input_layernorm.weight] Loading weights: 86%|████████▌ | 267/311 [00:01<00:00, 327.90it/s, Materializing param=model.layers.24.input_layernorm.weight] Loading weights: 99%|█████████▉| 308/311 [00:01<00:00, 383.44it/s, Materializing param=model.layers.27.self_attn.q_norm.weight] Loading weights: 86%|████████▌ | 268/311 [00:01<00:00, 327.90it/s, Materializing param=model.layers.24.mlp.down_proj.weight] Loading weights: 86%|████████▌ | 268/311 [00:01<00:00, 327.90it/s, Materializing param=model.layers.24.mlp.down_proj.weight] Loading weights: 99%|█████████▉| 308/311 [00:01<00:00, 383.44it/s, Materializing param=model.layers.27.self_attn.q_norm.weight] Loading weights: 99%|█████████▉| 309/311 [00:01<00:00, 383.44it/s, Materializing param=model.layers.27.self_attn.q_proj.weight] Loading weights: 99%|█████████▉| 309/311 [00:01<00:00, 383.44it/s, Materializing param=model.layers.27.self_attn.q_proj.weight] Loading weights: 100%|█████████▉| 310/311 [00:01<00:00, 383.44it/s, Materializing param=model.layers.27.self_attn.v_proj.weight] Loading weights: 100%|█████████▉| 310/311 [00:01<00:00, 383.44it/s, Materializing param=model.layers.27.self_attn.v_proj.weight]The tied weights mapping and config for this model specifies to tie model.embed_tokens.weight to lm_head.weight, but both are present in the checkpoints, so we will NOT tie them. You should update the config with `tie_word_embeddings=False` to silence this warning + Loading weights: 100%|██████████| 311/311 [00:01<00:00, 383.44it/s, Materializing param=model.norm.weight] Loading weights: 100%|██████████| 311/311 [00:01<00:00, 383.44it/s, Materializing param=model.norm.weight] Loading weights: 100%|██████████| 311/311 [00:01<00:00, 300.46it/s, Materializing param=model.norm.weight] + Loading weights: 73%|███████▎ | 226/311 [00:00<00:00, 329.16it/s, Materializing param=model.layers.20.mlp.up_proj.weight] Loading weights: 73%|███████▎ | 226/311 [00:00<00:00, 329.16it/s, Materializing param=model.layers.20.mlp.up_proj.weight]The tied weights mapping and config for this model specifies to tie model.embed_tokens.weight to lm_head.weight, but both are present in the checkpoints, so we will NOT tie them. You should update the config with `tie_word_embeddings=False` to silence this warning + Loading weights: 80%|███████▉ | 248/311 [00:01<00:00, 334.15it/s, Materializing param=model.layers.22.mlp.up_proj.weight] Loading weights: 80%|███████▉ | 248/311 [00:01<00:00, 334.15it/s, Materializing param=model.layers.22.mlp.up_proj.weight] Loading weights: 73%|███████▎ | 227/311 [00:00<00:00, 329.16it/s, Materializing param=model.layers.20.post_attention_layernorm.weight] Loading weights: 73%|███████▎ | 227/311 [00:00<00:00, 329.16it/s, Materializing param=model.layers.20.post_attention_layernorm.weight] Loading weights: 73%|███████▎ | 228/311 [00:00<00:00, 329.16it/s, Materializing param=model.layers.20.self_attn.k_norm.weight] Loading weights: 73%|███████▎ | 228/311 [00:00<00:00, 329.16it/s, Materializing param=model.layers.20.self_attn.k_norm.weight] Loading weights: 74%|███████▎ | 229/311 [00:00<00:00, 329.16it/s, Materializing param=model.layers.20.self_attn.k_proj.weight] Loading weights: 74%|███████▎ | 229/311 [00:00<00:00, 329.16it/s, Materializing param=model.layers.20.self_attn.k_proj.weight] Loading weights: 74%|███████▍ | 230/311 [00:00<00:00, 329.16it/s, Materializing param=model.layers.20.self_attn.o_proj.weight] Loading weights: 74%|███████▍ | 230/311 [00:00<00:00, 329.16it/s, Materializing param=model.layers.20.self_attn.o_proj.weight] Loading weights: 80%|████████ | 249/311 [00:01<00:00, 334.15it/s, Materializing param=model.layers.22.post_attention_layernorm.weight] Loading weights: 80%|████████ | 249/311 [00:01<00:00, 334.15it/s, Materializing param=model.layers.22.post_attention_layernorm.weight] Loading weights: 80%|████████ | 250/311 [00:01<00:00, 334.15it/s, Materializing param=model.layers.22.self_attn.k_norm.weight] Loading weights: 80%|████████ | 250/311 [00:01<00:00, 334.15it/s, Materializing param=model.layers.22.self_attn.k_norm.weight] Loading weights: 86%|████████▋ | 269/311 [00:01<00:00, 347.39it/s, Materializing param=model.layers.24.mlp.down_proj.weight] Loading weights: 81%|████████ | 251/311 [00:01<00:00, 334.15it/s, Materializing param=model.layers.22.self_attn.k_proj.weight] Loading weights: 86%|████████▋ | 269/311 [00:01<00:00, 347.39it/s, Materializing param=model.layers.24.mlp.gate_proj.weight] Loading weights: 81%|████████ | 251/311 [00:01<00:00, 334.15it/s, Materializing param=model.layers.22.self_attn.k_proj.weight] Loading weights: 86%|████████▋ | 269/311 [00:01<00:00, 347.39it/s, Materializing param=model.layers.24.mlp.gate_proj.weight] Loading weights: 87%|████████▋ | 270/311 [00:01<00:00, 347.39it/s, Materializing param=model.layers.24.mlp.up_proj.weight] Loading weights: 87%|████████▋ | 270/311 [00:01<00:00, 347.39it/s, Materializing param=model.layers.24.mlp.up_proj.weight] Loading weights: 74%|███████▍ | 231/311 [00:00<00:00, 329.16it/s, Materializing param=model.layers.20.self_attn.q_norm.weight] Loading weights: 74%|███████▍ | 231/311 [00:00<00:00, 329.16it/s, Materializing param=model.layers.20.self_attn.q_norm.weight] Loading weights: 75%|███████▍ | 232/311 [00:00<00:00, 329.16it/s, Materializing param=model.layers.20.self_attn.q_proj.weight] Loading weights: 81%|████████ | 252/311 [00:01<00:00, 334.15it/s, Materializing param=model.layers.22.self_attn.o_proj.weight] Loading weights: 75%|███████▍ | 232/311 [00:00<00:00, 329.16it/s, Materializing param=model.layers.20.self_attn.q_proj.weight] Loading weights: 81%|████████ | 252/311 [00:01<00:00, 334.15it/s, Materializing param=model.layers.22.self_attn.o_proj.weight] Loading weights: 75%|███████▍ | 233/311 [00:00<00:00, 329.16it/s, Materializing param=model.layers.20.self_attn.v_proj.weight] Loading weights: 75%|███████▍ | 233/311 [00:00<00:00, 329.16it/s, Materializing param=model.layers.20.self_attn.v_proj.weight] Loading weights: 81%|████████▏ | 253/311 [00:01<00:00, 334.15it/s, Materializing param=model.layers.22.self_attn.q_norm.weight] Loading weights: 81%|████████▏ | 253/311 [00:01<00:00, 334.15it/s, Materializing param=model.layers.22.self_attn.q_norm.weight] Loading weights: 82%|████████▏ | 254/311 [00:01<00:00, 334.15it/s, Materializing param=model.layers.22.self_attn.q_proj.weight] Loading weights: 82%|████████▏ | 254/311 [00:01<00:00, 334.15it/s, Materializing param=model.layers.22.self_attn.q_proj.weight] Loading weights: 75%|███████▌ | 234/311 [00:00<00:00, 329.16it/s, Materializing param=model.layers.21.input_layernorm.weight] Loading weights: 75%|███████▌ | 234/311 [00:00<00:00, 329.16it/s, Materializing param=model.layers.21.input_layernorm.weight] Loading weights: 76%|███████▌ | 235/311 [00:00<00:00, 329.16it/s, Materializing param=model.layers.21.mlp.down_proj.weight] Loading weights: 76%|███████▌ | 235/311 [00:00<00:00, 329.16it/s, Materializing param=model.layers.21.mlp.down_proj.weight] Loading weights: 82%|████████▏ | 255/311 [00:01<00:00, 334.15it/s, Materializing param=model.layers.22.self_attn.v_proj.weight] Loading weights: 82%|████████▏ | 255/311 [00:01<00:00, 334.15it/s, Materializing param=model.layers.22.self_attn.v_proj.weight] Loading weights: 87%|████████▋ | 271/311 [00:01<00:00, 347.39it/s, Materializing param=model.layers.24.post_attention_layernorm.weight] Loading weights: 87%|████████▋ | 271/311 [00:01<00:00, 347.39it/s, Materializing param=model.layers.24.post_attention_layernorm.weight] Loading weights: 82%|████████▏ | 256/311 [00:01<00:00, 334.15it/s, Materializing param=model.layers.23.input_layernorm.weight] Loading weights: 82%|████████▏ | 256/311 [00:01<00:00, 334.15it/s, Materializing param=model.layers.23.input_layernorm.weight] Loading weights: 83%|████████▎ | 257/311 [00:01<00:00, 334.15it/s, Materializing param=model.layers.23.mlp.down_proj.weight] Loading weights: 83%|████████▎ | 257/311 [00:01<00:00, 334.15it/s, Materializing param=model.layers.23.mlp.down_proj.weight] Loading weights: 87%|████████▋ | 272/311 [00:01<00:00, 347.39it/s, Materializing param=model.layers.24.self_attn.k_norm.weight] Loading weights: 87%|████████▋ | 272/311 [00:01<00:00, 347.39it/s, Materializing param=model.layers.24.self_attn.k_norm.weight] Loading weights: 88%|████████▊ | 273/311 [00:01<00:00, 347.39it/s, Materializing param=model.layers.24.self_attn.k_proj.weight] Loading weights: 88%|████████▊ | 273/311 [00:01<00:00, 347.39it/s, Materializing param=model.layers.24.self_attn.k_proj.weight] Loading weights: 88%|████████▊ | 274/311 [00:01<00:00, 347.39it/s, Materializing param=model.layers.24.self_attn.o_proj.weight] Loading weights: 88%|████████▊ | 274/311 [00:01<00:00, 347.39it/s, Materializing param=model.layers.24.self_attn.o_proj.weight] Loading weights: 88%|████████▊ | 275/311 [00:01<00:00, 347.39it/s, Materializing param=model.layers.24.self_attn.q_norm.weight] Loading weights: 88%|████████▊ | 275/311 [00:01<00:00, 347.39it/s, Materializing param=model.layers.24.self_attn.q_norm.weight] Loading weights: 89%|████████▊ | 276/311 [00:01<00:00, 347.39it/s, Materializing param=model.layers.24.self_attn.q_proj.weight] Loading weights: 89%|████████▊ | 276/311 [00:01<00:00, 347.39it/s, Materializing param=model.layers.24.self_attn.q_proj.weight] Loading weights: 76%|███████▌ | 236/311 [00:00<00:00, 329.16it/s, Materializing param=model.layers.21.mlp.gate_proj.weight] Loading weights: 76%|███████▌ | 236/311 [00:00<00:00, 329.16it/s, Materializing param=model.layers.21.mlp.gate_proj.weight] Loading weights: 89%|████████▉ | 277/311 [00:01<00:00, 347.39it/s, Materializing param=model.layers.24.self_attn.v_proj.weight] Loading weights: 89%|████████▉ | 277/311 [00:01<00:00, 347.39it/s, Materializing param=model.layers.24.self_attn.v_proj.weight] Loading weights: 89%|████████▉ | 278/311 [00:01<00:00, 347.39it/s, Materializing param=model.layers.25.input_layernorm.weight] Loading weights: 89%|████████▉ | 278/311 [00:01<00:00, 347.39it/s, Materializing param=model.layers.25.input_layernorm.weight] Loading weights: 90%|████████▉ | 279/311 [00:01<00:00, 347.39it/s, Materializing param=model.layers.25.mlp.down_proj.weight] Loading weights: 90%|████████▉ | 279/311 [00:01<00:00, 347.39it/s, Materializing param=model.layers.25.mlp.down_proj.weight] Loading weights: 83%|████████▎ | 258/311 [00:01<00:00, 334.15it/s, Materializing param=model.layers.23.mlp.gate_proj.weight] Loading weights: 83%|████████▎ | 258/311 [00:01<00:00, 334.15it/s, Materializing param=model.layers.23.mlp.gate_proj.weight] Loading weights: 76%|███████▌ | 237/311 [00:00<00:00, 329.16it/s, Materializing param=model.layers.21.mlp.up_proj.weight] Loading weights: 76%|███████▌ | 237/311 [00:00<00:00, 329.16it/s, Materializing param=model.layers.21.mlp.up_proj.weight] Loading weights: 90%|█████████ | 280/311 [00:01<00:00, 347.39it/s, Materializing param=model.layers.25.mlp.gate_proj.weight] Loading weights: 90%|█████████ | 280/311 [00:01<00:00, 347.39it/s, Materializing param=model.layers.25.mlp.gate_proj.weight] Loading weights: 83%|████████▎ | 259/311 [00:01<00:00, 334.15it/s, Materializing param=model.layers.23.mlp.up_proj.weight] Loading weights: 83%|████████▎ | 259/311 [00:01<00:00, 334.15it/s, Materializing param=model.layers.23.mlp.up_proj.weight] Loading weights: 77%|███████▋ | 238/311 [00:00<00:00, 345.75it/s, Materializing param=model.layers.21.mlp.up_proj.weight] Loading weights: 77%|███████▋ | 238/311 [00:00<00:00, 345.75it/s, Materializing param=model.layers.21.post_attention_layernorm.weight] Loading weights: 77%|███████▋ | 238/311 [00:00<00:00, 345.75it/s, Materializing param=model.layers.21.post_attention_layernorm.weight] Loading weights: 77%|███████▋ | 239/311 [00:00<00:00, 345.75it/s, Materializing param=model.layers.21.self_attn.k_norm.weight] Loading weights: 77%|███████▋ | 239/311 [00:00<00:00, 345.75it/s, Materializing param=model.layers.21.self_attn.k_norm.weight] Loading weights: 77%|███████▋ | 240/311 [00:00<00:00, 345.75it/s, Materializing param=model.layers.21.self_attn.k_proj.weight] Loading weights: 77%|███████▋ | 240/311 [00:00<00:00, 345.75it/s, Materializing param=model.layers.21.self_attn.k_proj.weight] Loading weights: 77%|███████▋ | 241/311 [00:00<00:00, 345.75it/s, Materializing param=model.layers.21.self_attn.o_proj.weight] Loading weights: 77%|███████▋ | 241/311 [00:00<00:00, 345.75it/s, Materializing param=model.layers.21.self_attn.o_proj.weight] Loading weights: 78%|███████▊ | 242/311 [00:00<00:00, 345.75it/s, Materializing param=model.layers.21.self_attn.q_norm.weight] Loading weights: 78%|███████▊ | 242/311 [00:00<00:00, 345.75it/s, Materializing param=model.layers.21.self_attn.q_norm.weight] Loading weights: 78%|███████▊ | 243/311 [00:00<00:00, 345.75it/s, Materializing param=model.layers.21.self_attn.q_proj.weight] Loading weights: 78%|███████▊ | 243/311 [00:00<00:00, 345.75it/s, Materializing param=model.layers.21.self_attn.q_proj.weight] Loading weights: 90%|█████████ | 281/311 [00:01<00:00, 347.39it/s, Materializing param=model.layers.25.mlp.up_proj.weight] Loading weights: 90%|█████████ | 281/311 [00:01<00:00, 347.39it/s, Materializing param=model.layers.25.mlp.up_proj.weight] Loading weights: 84%|████████▎ | 260/311 [00:01<00:00, 334.15it/s, Materializing param=model.layers.23.post_attention_layernorm.weight] Loading weights: 84%|████████▎ | 260/311 [00:01<00:00, 334.15it/s, Materializing param=model.layers.23.post_attention_layernorm.weight] Loading weights: 84%|████████▍ | 261/311 [00:01<00:00, 334.15it/s, Materializing param=model.layers.23.self_attn.k_norm.weight] Loading weights: 84%|████████▍ | 261/311 [00:01<00:00, 334.15it/s, Materializing param=model.layers.23.self_attn.k_norm.weight] Loading weights: 84%|████████▍ | 262/311 [00:01<00:00, 334.15it/s, Materializing param=model.layers.23.self_attn.k_proj.weight] Loading weights: 84%|████████▍ | 262/311 [00:01<00:00, 334.15it/s, Materializing param=model.layers.23.self_attn.k_proj.weight] Loading weights: 78%|███████▊ | 244/311 [00:00<00:00, 345.75it/s, Materializing param=model.layers.21.self_attn.v_proj.weight] Loading weights: 78%|███████▊ | 244/311 [00:00<00:00, 345.75it/s, Materializing param=model.layers.21.self_attn.v_proj.weight] Loading weights: 85%|████████▍ | 263/311 [00:01<00:00, 334.15it/s, Materializing param=model.layers.23.self_attn.o_proj.weight] Loading weights: 85%|████████▍ | 263/311 [00:01<00:00, 334.15it/s, Materializing param=model.layers.23.self_attn.o_proj.weight] Loading weights: 79%|███████▉ | 245/311 [00:00<00:00, 345.75it/s, Materializing param=model.layers.22.input_layernorm.weight] Loading weights: 79%|███████▉ | 245/311 [00:00<00:00, 345.75it/s, Materializing param=model.layers.22.input_layernorm.weight] Loading weights: 85%|████████▍ | 264/311 [00:01<00:00, 334.15it/s, Materializing param=model.layers.23.self_attn.q_norm.weight] Loading weights: 85%|████████▍ | 264/311 [00:01<00:00, 334.15it/s, Materializing param=model.layers.23.self_attn.q_norm.weight] Loading weights: 85%|████████▌ | 265/311 [00:01<00:00, 334.15it/s, Materializing param=model.layers.23.self_attn.q_proj.weight] Loading weights: 85%|████████▌ | 265/311 [00:01<00:00, 334.15it/s, Materializing param=model.layers.23.self_attn.q_proj.weight] Loading weights: 86%|████████▌ | 266/311 [00:01<00:00, 334.15it/s, Materializing param=model.layers.23.self_attn.v_proj.weight] Loading weights: 86%|████████▌ | 266/311 [00:01<00:00, 334.15it/s, Materializing param=model.layers.23.self_attn.v_proj.weight] Loading weights: 91%|█████████ | 282/311 [00:01<00:00, 347.39it/s, Materializing param=model.layers.25.post_attention_layernorm.weight] Loading weights: 91%|█████████ | 282/311 [00:01<00:00, 347.39it/s, Materializing param=model.layers.25.post_attention_layernorm.weight] Loading weights: 91%|█████████ | 283/311 [00:01<00:00, 347.39it/s, Materializing param=model.layers.25.self_attn.k_norm.weight] Loading weights: 91%|█████████ | 283/311 [00:01<00:00, 347.39it/s, Materializing param=model.layers.25.self_attn.k_norm.weight] Loading weights: 91%|█████████▏| 284/311 [00:01<00:00, 347.39it/s, Materializing param=model.layers.25.self_attn.k_proj.weight] Loading weights: 91%|█████████▏| 284/311 [00:01<00:00, 347.39it/s, Materializing param=model.layers.25.self_attn.k_proj.weight] Loading weights: 92%|█████████▏| 285/311 [00:01<00:00, 347.39it/s, Materializing param=model.layers.25.self_attn.o_proj.weight] Loading weights: 92%|█████████▏| 285/311 [00:01<00:00, 347.39it/s, Materializing param=model.layers.25.self_attn.o_proj.weight] Loading weights: 86%|████████▌ | 267/311 [00:01<00:00, 334.15it/s, Materializing param=model.layers.24.input_layernorm.weight] Loading weights: 86%|████████▌ | 267/311 [00:01<00:00, 334.15it/s, Materializing param=model.layers.24.input_layernorm.weight] Loading weights: 86%|████████▌ | 268/311 [00:01<00:00, 334.15it/s, Materializing param=model.layers.24.mlp.down_proj.weight] Loading weights: 86%|████████▌ | 268/311 [00:01<00:00, 334.15it/s, Materializing param=model.layers.24.mlp.down_proj.weight] Loading weights: 79%|███████▉ | 246/311 [00:00<00:00, 345.75it/s, Materializing param=model.layers.22.mlp.down_proj.weight] Loading weights: 79%|███████▉ | 246/311 [00:00<00:00, 345.75it/s, Materializing param=model.layers.22.mlp.down_proj.weight] Loading weights: 79%|███████▉ | 247/311 [00:00<00:00, 345.75it/s, Materializing param=model.layers.22.mlp.gate_proj.weight] Loading weights: 79%|███████▉ | 247/311 [00:00<00:00, 345.75it/s, Materializing param=model.layers.22.mlp.gate_proj.weight] Loading weights: 92%|█████████▏| 286/311 [00:01<00:00, 347.39it/s, Materializing param=model.layers.25.self_attn.q_norm.weight] Loading weights: 92%|█████████▏| 286/311 [00:01<00:00, 347.39it/s, Materializing param=model.layers.25.self_attn.q_norm.weight] Loading weights: 92%|█████████▏| 287/311 [00:01<00:00, 347.39it/s, Materializing param=model.layers.25.self_attn.q_proj.weight] Loading weights: 92%|█████████▏| 287/311 [00:01<00:00, 347.39it/s, Materializing param=model.layers.25.self_attn.q_proj.weight] Loading weights: 93%|█████████▎| 288/311 [00:01<00:00, 347.39it/s, Materializing param=model.layers.25.self_attn.v_proj.weight] Loading weights: 93%|█████████▎| 288/311 [00:01<00:00, 347.39it/s, Materializing param=model.layers.25.self_attn.v_proj.weight] Loading weights: 93%|█████████▎| 289/311 [00:01<00:00, 347.39it/s, Materializing param=model.layers.26.input_layernorm.weight] Loading weights: 93%|█████████▎| 289/311 [00:01<00:00, 347.39it/s, Materializing param=model.layers.26.input_layernorm.weight] Loading weights: 93%|█████████▎| 290/311 [00:01<00:00, 347.39it/s, Materializing param=model.layers.26.mlp.down_proj.weight] Loading weights: 93%|█████████▎| 290/311 [00:01<00:00, 347.39it/s, Materializing param=model.layers.26.mlp.down_proj.weight] Loading weights: 86%|████████▋ | 269/311 [00:01<00:00, 334.15it/s, Materializing param=model.layers.24.mlp.gate_proj.weight] Loading weights: 86%|████████▋ | 269/311 [00:01<00:00, 334.15it/s, Materializing param=model.layers.24.mlp.gate_proj.weight] Loading weights: 80%|███████▉ | 248/311 [00:00<00:00, 345.75it/s, Materializing param=model.layers.22.mlp.up_proj.weight] Loading weights: 80%|███████▉ | 248/311 [00:00<00:00, 345.75it/s, Materializing param=model.layers.22.mlp.up_proj.weight] Loading weights: 94%|█████████▎| 291/311 [00:01<00:00, 347.39it/s, Materializing param=model.layers.26.mlp.gate_proj.weight] Loading weights: 94%|█████████▎| 291/311 [00:01<00:00, 347.39it/s, Materializing param=model.layers.26.mlp.gate_proj.weight] Loading weights: 87%|████████▋ | 270/311 [00:01<00:00, 348.48it/s, Materializing param=model.layers.24.mlp.gate_proj.weight] Loading weights: 87%|████████▋ | 270/311 [00:01<00:00, 348.48it/s, Materializing param=model.layers.24.mlp.up_proj.weight] Loading weights: 87%|████████▋ | 270/311 [00:01<00:00, 348.48it/s, Materializing param=model.layers.24.mlp.up_proj.weight] Loading weights: 80%|████████ | 249/311 [00:00<00:00, 345.75it/s, Materializing param=model.layers.22.post_attention_layernorm.weight] Loading weights: 80%|████████ | 249/311 [00:00<00:00, 345.75it/s, Materializing param=model.layers.22.post_attention_layernorm.weight] Loading weights: 80%|████████ | 250/311 [00:00<00:00, 345.75it/s, Materializing param=model.layers.22.self_attn.k_norm.weight] Loading weights: 80%|████████ | 250/311 [00:00<00:00, 345.75it/s, Materializing param=model.layers.22.self_attn.k_norm.weight] Loading weights: 81%|████████ | 251/311 [00:00<00:00, 345.75it/s, Materializing param=model.layers.22.self_attn.k_proj.weight] Loading weights: 81%|████████ | 251/311 [00:00<00:00, 345.75it/s, Materializing param=model.layers.22.self_attn.k_proj.weight] Loading weights: 81%|████████ | 252/311 [00:00<00:00, 345.75it/s, Materializing param=model.layers.22.self_attn.o_proj.weight] Loading weights: 81%|████████ | 252/311 [00:00<00:00, 345.75it/s, Materializing param=model.layers.22.self_attn.o_proj.weight] Loading weights: 81%|████████▏ | 253/311 [00:00<00:00, 345.75it/s, Materializing param=model.layers.22.self_attn.q_norm.weight] Loading weights: 81%|████████▏ | 253/311 [00:00<00:00, 345.75it/s, Materializing param=model.layers.22.self_attn.q_norm.weight]The tied weights mapping and config for this model specifies to tie model.embed_tokens.weight to lm_head.weight, but both are present in the checkpoints, so we will NOT tie them. You should update the config with `tie_word_embeddings=False` to silence this warning + Loading weights: 82%|████████▏ | 254/311 [00:00<00:00, 345.75it/s, Materializing param=model.layers.22.self_attn.q_proj.weight] Loading weights: 82%|████████▏ | 254/311 [00:00<00:00, 345.75it/s, Materializing param=model.layers.22.self_attn.q_proj.weight] Loading weights: 94%|█████████▍| 292/311 [00:01<00:00, 347.39it/s, Materializing param=model.layers.26.mlp.up_proj.weight] Loading weights: 94%|█████████▍| 292/311 [00:01<00:00, 347.39it/s, Materializing param=model.layers.26.mlp.up_proj.weight] Loading weights: 82%|████████▏ | 255/311 [00:00<00:00, 345.75it/s, Materializing param=model.layers.22.self_attn.v_proj.weight] Loading weights: 82%|████████▏ | 255/311 [00:00<00:00, 345.75it/s, Materializing param=model.layers.22.self_attn.v_proj.weight] Loading weights: 87%|████████▋ | 271/311 [00:01<00:00, 348.48it/s, Materializing param=model.layers.24.post_attention_layernorm.weight] Loading weights: 87%|████████▋ | 271/311 [00:01<00:00, 348.48it/s, Materializing param=model.layers.24.post_attention_layernorm.weight] Loading weights: 87%|████████▋ | 272/311 [00:01<00:00, 348.48it/s, Materializing param=model.layers.24.self_attn.k_norm.weight] Loading weights: 87%|████████▋ | 272/311 [00:01<00:00, 348.48it/s, Materializing param=model.layers.24.self_attn.k_norm.weight] Loading weights: 82%|████████▏ | 256/311 [00:00<00:00, 345.75it/s, Materializing param=model.layers.23.input_layernorm.weight] Loading weights: 82%|████████▏ | 256/311 [00:00<00:00, 345.75it/s, Materializing param=model.layers.23.input_layernorm.weight] Loading weights: 88%|████████▊ | 273/311 [00:01<00:00, 348.48it/s, Materializing param=model.layers.24.self_attn.k_proj.weight] Loading weights: 88%|████████▊ | 273/311 [00:01<00:00, 348.48it/s, Materializing param=model.layers.24.self_attn.k_proj.weight] Loading weights: 83%|████████▎ | 257/311 [00:00<00:00, 345.75it/s, Materializing param=model.layers.23.mlp.down_proj.weight] Loading weights: 83%|████████▎ | 257/311 [00:00<00:00, 345.75it/s, Materializing param=model.layers.23.mlp.down_proj.weight] Loading weights: 88%|████████▊ | 274/311 [00:01<00:00, 348.48it/s, Materializing param=model.layers.24.self_attn.o_proj.weight] Loading weights: 88%|████████▊ | 274/311 [00:01<00:00, 348.48it/s, Materializing param=model.layers.24.self_attn.o_proj.weight] Loading weights: 88%|████████▊ | 275/311 [00:01<00:00, 348.48it/s, Materializing param=model.layers.24.self_attn.q_norm.weight] Loading weights: 88%|████████▊ | 275/311 [00:01<00:00, 348.48it/s, Materializing param=model.layers.24.self_attn.q_norm.weight]The tied weights mapping and config for this model specifies to tie model.embed_tokens.weight to lm_head.weight, but both are present in the checkpoints, so we will NOT tie them. You should update the config with `tie_word_embeddings=False` to silence this warning + Loading weights: 94%|█████████▍| 293/311 [00:01<00:00, 347.39it/s, Materializing param=model.layers.26.post_attention_layernorm.weight] Loading weights: 94%|█████████▍| 293/311 [00:01<00:00, 347.39it/s, Materializing param=model.layers.26.post_attention_layernorm.weight] Loading weights: 95%|█████████▍| 294/311 [00:01<00:00, 347.39it/s, Materializing param=model.layers.26.self_attn.k_norm.weight] Loading weights: 95%|█████████▍| 294/311 [00:01<00:00, 347.39it/s, Materializing param=model.layers.26.self_attn.k_norm.weight] Loading weights: 95%|█████████▍| 295/311 [00:01<00:00, 347.39it/s, Materializing param=model.layers.26.self_attn.k_proj.weight] Loading weights: 95%|█████████▍| 295/311 [00:01<00:00, 347.39it/s, Materializing param=model.layers.26.self_attn.k_proj.weight] Loading weights: 89%|████████▊ | 276/311 [00:01<00:00, 348.48it/s, Materializing param=model.layers.24.self_attn.q_proj.weight] Loading weights: 89%|████████▊ | 276/311 [00:01<00:00, 348.48it/s, Materializing param=model.layers.24.self_attn.q_proj.weight] Loading weights: 89%|████████▉ | 277/311 [00:01<00:00, 348.48it/s, Materializing param=model.layers.24.self_attn.v_proj.weight] Loading weights: 89%|████████▉ | 277/311 [00:01<00:00, 348.48it/s, Materializing param=model.layers.24.self_attn.v_proj.weight] Loading weights: 89%|████████▉ | 278/311 [00:01<00:00, 348.48it/s, Materializing param=model.layers.25.input_layernorm.weight] Loading weights: 89%|████████▉ | 278/311 [00:01<00:00, 348.48it/s, Materializing param=model.layers.25.input_layernorm.weight] Loading weights: 83%|████████▎ | 258/311 [00:00<00:00, 345.75it/s, Materializing param=model.layers.23.mlp.gate_proj.weight] Loading weights: 90%|████████▉ | 279/311 [00:01<00:00, 348.48it/s, Materializing param=model.layers.25.mlp.down_proj.weight] Loading weights: 90%|████████▉ | 279/311 [00:01<00:00, 348.48it/s, Materializing param=model.layers.25.mlp.down_proj.weight] Loading weights: 83%|████████▎ | 258/311 [00:00<00:00, 345.75it/s, Materializing param=model.layers.23.mlp.gate_proj.weight] Loading weights: 95%|█████████▌| 296/311 [00:01<00:00, 347.39it/s, Materializing param=model.layers.26.self_attn.o_proj.weight] Loading weights: 95%|█████████▌| 296/311 [00:01<00:00, 347.39it/s, Materializing param=model.layers.26.self_attn.o_proj.weight] Loading weights: 95%|█████████▌| 297/311 [00:01<00:00, 347.39it/s, Materializing param=model.layers.26.self_attn.q_norm.weight] Loading weights: 95%|█████████▌| 297/311 [00:01<00:00, 347.39it/s, Materializing param=model.layers.26.self_attn.q_norm.weight] Loading weights: 96%|█████████▌| 298/311 [00:01<00:00, 347.39it/s, Materializing param=model.layers.26.self_attn.q_proj.weight] Loading weights: 96%|█████████▌| 298/311 [00:01<00:00, 347.39it/s, Materializing param=model.layers.26.self_attn.q_proj.weight] Loading weights: 96%|█████████▌| 299/311 [00:01<00:00, 347.39it/s, Materializing param=model.layers.26.self_attn.v_proj.weight] Loading weights: 96%|█████████▌| 299/311 [00:01<00:00, 347.39it/s, Materializing param=model.layers.26.self_attn.v_proj.weight] Loading weights: 96%|█████████▋| 300/311 [00:01<00:00, 347.39it/s, Materializing param=model.layers.27.input_layernorm.weight] Loading weights: 96%|█████████▋| 300/311 [00:01<00:00, 347.39it/s, Materializing param=model.layers.27.input_layernorm.weight] Loading weights: 90%|█████████ | 280/311 [00:01<00:00, 348.48it/s, Materializing param=model.layers.25.mlp.gate_proj.weight] Loading weights: 90%|█████████ | 280/311 [00:01<00:00, 348.48it/s, Materializing param=model.layers.25.mlp.gate_proj.weight] Loading weights: 83%|████████▎ | 259/311 [00:01<00:00, 345.75it/s, Materializing param=model.layers.23.mlp.up_proj.weight] Loading weights: 83%|████████▎ | 259/311 [00:01<00:00, 345.75it/s, Materializing param=model.layers.23.mlp.up_proj.weight] Loading weights: 97%|█████████▋| 301/311 [00:01<00:00, 347.39it/s, Materializing param=model.layers.27.mlp.down_proj.weight] Loading weights: 97%|█████████▋| 301/311 [00:01<00:00, 347.39it/s, Materializing param=model.layers.27.mlp.down_proj.weight] Loading weights: 97%|█████████▋| 302/311 [00:01<00:00, 347.39it/s, Materializing param=model.layers.27.mlp.gate_proj.weight] Loading weights: 97%|█████████▋| 302/311 [00:01<00:00, 347.39it/s, Materializing param=model.layers.27.mlp.gate_proj.weight] Loading weights: 84%|████████▎ | 260/311 [00:01<00:00, 345.75it/s, Materializing param=model.layers.23.post_attention_layernorm.weight] Loading weights: 84%|████████▎ | 260/311 [00:01<00:00, 345.75it/s, Materializing param=model.layers.23.post_attention_layernorm.weight] Loading weights: 84%|████████▍ | 261/311 [00:01<00:00, 345.75it/s, Materializing param=model.layers.23.self_attn.k_norm.weight] Loading weights: 84%|████████▍ | 261/311 [00:01<00:00, 345.75it/s, Materializing param=model.layers.23.self_attn.k_norm.weight] Loading weights: 84%|████████▍ | 262/311 [00:01<00:00, 345.75it/s, Materializing param=model.layers.23.self_attn.k_proj.weight] Loading weights: 84%|████████▍ | 262/311 [00:01<00:00, 345.75it/s, Materializing param=model.layers.23.self_attn.k_proj.weight] Loading weights: 90%|█████████ | 281/311 [00:01<00:00, 348.48it/s, Materializing param=model.layers.25.mlp.up_proj.weight] Loading weights: 85%|████████▍ | 263/311 [00:01<00:00, 345.75it/s, Materializing param=model.layers.23.self_attn.o_proj.weight] Loading weights: 90%|█████████ | 281/311 [00:01<00:00, 348.48it/s, Materializing param=model.layers.25.mlp.up_proj.weight] Loading weights: 85%|████████▍ | 263/311 [00:01<00:00, 345.75it/s, Materializing param=model.layers.23.self_attn.o_proj.weight]ywang29-p4d-debug-1-worker-0:243411:243411 [0] NCCL INFO Comm config Blocking set to 1 + Loading weights: 85%|████████▍ | 264/311 [00:01<00:00, 345.75it/s, Materializing param=model.layers.23.self_attn.q_norm.weight] Loading weights: 85%|████████▍ | 264/311 [00:01<00:00, 345.75it/s, Materializing param=model.layers.23.self_attn.q_norm.weight] Loading weights: 97%|█████████▋| 303/311 [00:01<00:00, 347.39it/s, Materializing param=model.layers.27.mlp.up_proj.weight] Loading weights: 97%|█████████▋| 303/311 [00:01<00:00, 347.39it/s, Materializing param=model.layers.27.mlp.up_proj.weight] Loading weights: 85%|████████▌ | 265/311 [00:01<00:00, 345.75it/s, Materializing param=model.layers.23.self_attn.q_proj.weight] Loading weights: 85%|████████▌ | 265/311 [00:01<00:00, 345.75it/s, Materializing param=model.layers.23.self_attn.q_proj.weight] Loading weights: 86%|████████▌ | 266/311 [00:01<00:00, 345.75it/s, Materializing param=model.layers.23.self_attn.v_proj.weight] Loading weights: 86%|████████▌ | 266/311 [00:01<00:00, 345.75it/s, Materializing param=model.layers.23.self_attn.v_proj.weight] Loading weights: 86%|████████▌ | 267/311 [00:01<00:00, 345.75it/s, Materializing param=model.layers.24.input_layernorm.weight] Loading weights: 86%|████████▌ | 267/311 [00:01<00:00, 345.75it/s, Materializing param=model.layers.24.input_layernorm.weight] Loading weights: 86%|████████▌ | 268/311 [00:01<00:00, 345.75it/s, Materializing param=model.layers.24.mlp.down_proj.weight] Loading weights: 86%|████████▌ | 268/311 [00:01<00:00, 345.75it/s, Materializing param=model.layers.24.mlp.down_proj.weight] Loading weights: 91%|█████████ | 282/311 [00:01<00:00, 348.48it/s, Materializing param=model.layers.25.post_attention_layernorm.weight] Loading weights: 91%|█████████ | 282/311 [00:01<00:00, 348.48it/s, Materializing param=model.layers.25.post_attention_layernorm.weight] Loading weights: 91%|█████████ | 283/311 [00:01<00:00, 348.48it/s, Materializing param=model.layers.25.self_attn.k_norm.weight] Loading weights: 91%|█████████ | 283/311 [00:01<00:00, 348.48it/s, Materializing param=model.layers.25.self_attn.k_norm.weight] Loading weights: 91%|█████████▏| 284/311 [00:01<00:00, 348.48it/s, Materializing param=model.layers.25.self_attn.k_proj.weight] Loading weights: 91%|█████████▏| 284/311 [00:01<00:00, 348.48it/s, Materializing param=model.layers.25.self_attn.k_proj.weight] Loading weights: 92%|█████████▏| 285/311 [00:01<00:00, 348.48it/s, Materializing param=model.layers.25.self_attn.o_proj.weight] Loading weights: 92%|█████████▏| 285/311 [00:01<00:00, 348.48it/s, Materializing param=model.layers.25.self_attn.o_proj.weight] Loading weights: 98%|█████████▊| 304/311 [00:01<00:00, 347.39it/s, Materializing param=model.layers.27.post_attention_layernorm.weight] Loading weights: 92%|█████████▏| 286/311 [00:01<00:00, 348.48it/s, Materializing param=model.layers.25.self_attn.q_norm.weight] Loading weights: 98%|█████████▊| 304/311 [00:01<00:00, 347.39it/s, Materializing param=model.layers.27.post_attention_layernorm.weight] Loading weights: 92%|█████████▏| 286/311 [00:01<00:00, 348.48it/s, Materializing param=model.layers.25.self_attn.q_norm.weight] Loading weights: 98%|█████████▊| 305/311 [00:01<00:00, 347.39it/s, Materializing param=model.layers.27.self_attn.k_norm.weight] Loading weights: 92%|█████████▏| 287/311 [00:01<00:00, 348.48it/s, Materializing param=model.layers.25.self_attn.q_proj.weight] Loading weights: 92%|█████████▏| 287/311 [00:01<00:00, 348.48it/s, Materializing param=model.layers.25.self_attn.q_proj.weight] Loading weights: 98%|█████████▊| 305/311 [00:01<00:00, 347.39it/s, Materializing param=model.layers.27.self_attn.k_norm.weight] Loading weights: 98%|█████████▊| 306/311 [00:01<00:00, 347.39it/s, Materializing param=model.layers.27.self_attn.k_proj.weight] Loading weights: 98%|█████████▊| 306/311 [00:01<00:00, 347.39it/s, Materializing param=model.layers.27.self_attn.k_proj.weight] Loading weights: 99%|█████████▊| 307/311 [00:01<00:00, 347.39it/s, Materializing param=model.layers.27.self_attn.o_proj.weight] Loading weights: 99%|█████████▊| 307/311 [00:01<00:00, 347.39it/s, Materializing param=model.layers.27.self_attn.o_proj.weight] Loading weights: 93%|█████████▎| 288/311 [00:01<00:00, 348.48it/s, Materializing param=model.layers.25.self_attn.v_proj.weight] Loading weights: 93%|█████████▎| 288/311 [00:01<00:00, 348.48it/s, Materializing param=model.layers.25.self_attn.v_proj.weight] Loading weights: 99%|█████████▉| 308/311 [00:01<00:00, 347.39it/s, Materializing param=model.layers.27.self_attn.q_norm.weight] Loading weights: 99%|█████████▉| 308/311 [00:01<00:00, 347.39it/s, Materializing param=model.layers.27.self_attn.q_norm.weight] Loading weights: 93%|█████████▎| 289/311 [00:01<00:00, 348.48it/s, Materializing param=model.layers.26.input_layernorm.weight] Loading weights: 93%|█████████▎| 289/311 [00:01<00:00, 348.48it/s, Materializing param=model.layers.26.input_layernorm.weight] Loading weights: 93%|█████████▎| 290/311 [00:01<00:00, 348.48it/s, Materializing param=model.layers.26.mlp.down_proj.weight] Loading weights: 93%|█████████▎| 290/311 [00:01<00:00, 348.48it/s, Materializing param=model.layers.26.mlp.down_proj.weight] Loading weights: 99%|█████████▉| 309/311 [00:01<00:00, 347.39it/s, Materializing param=model.layers.27.self_attn.q_proj.weight] Loading weights: 99%|█████████▉| 309/311 [00:01<00:00, 347.39it/s, Materializing param=model.layers.27.self_attn.q_proj.weight] Loading weights: 100%|█████████▉| 310/311 [00:01<00:00, 347.39it/s, Materializing param=model.layers.27.self_attn.v_proj.weight] Loading weights: 100%|█████████▉| 310/311 [00:01<00:00, 347.39it/s, Materializing param=model.layers.27.self_attn.v_proj.weight] Loading weights: 100%|██████████| 311/311 [00:01<00:00, 347.39it/s, Materializing param=model.norm.weight] Loading weights: 100%|██████████| 311/311 [00:01<00:00, 347.39it/s, Materializing param=model.norm.weight] Loading weights: 100%|██████████| 311/311 [00:01<00:00, 272.27it/s, Materializing param=model.norm.weight] + Loading weights: 86%|████████▋ | 269/311 [00:01<00:00, 345.75it/s, Materializing param=model.layers.24.mlp.gate_proj.weight] Loading weights: 86%|████████▋ | 269/311 [00:01<00:00, 345.75it/s, Materializing param=model.layers.24.mlp.gate_proj.weight] Loading weights: 87%|████████▋ | 270/311 [00:01<00:00, 345.75it/s, Materializing param=model.layers.24.mlp.up_proj.weight] Loading weights: 87%|████████▋ | 270/311 [00:01<00:00, 345.75it/s, Materializing param=model.layers.24.mlp.up_proj.weight] Loading weights: 94%|█████████▎| 291/311 [00:01<00:00, 348.48it/s, Materializing param=model.layers.26.mlp.gate_proj.weight] Loading weights: 94%|█████████▎| 291/311 [00:01<00:00, 348.48it/s, Materializing param=model.layers.26.mlp.gate_proj.weight] Loading weights: 87%|████████▋ | 271/311 [00:01<00:00, 345.75it/s, Materializing param=model.layers.24.post_attention_layernorm.weight] Loading weights: 87%|████████▋ | 271/311 [00:01<00:00, 345.75it/s, Materializing param=model.layers.24.post_attention_layernorm.weight] Loading weights: 87%|████████▋ | 272/311 [00:01<00:00, 345.75it/s, Materializing param=model.layers.24.self_attn.k_norm.weight] Loading weights: 87%|████████▋ | 272/311 [00:01<00:00, 345.75it/s, Materializing param=model.layers.24.self_attn.k_norm.weight] Loading weights: 88%|████████▊ | 273/311 [00:01<00:00, 345.75it/s, Materializing param=model.layers.24.self_attn.k_proj.weight] Loading weights: 88%|████████▊ | 273/311 [00:01<00:00, 345.75it/s, Materializing param=model.layers.24.self_attn.k_proj.weight] Loading weights: 88%|████████▊ | 274/311 [00:01<00:00, 345.75it/s, Materializing param=model.layers.24.self_attn.o_proj.weight] Loading weights: 88%|████████▊ | 274/311 [00:01<00:00, 345.75it/s, Materializing param=model.layers.24.self_attn.o_proj.weight] Loading weights: 94%|█████████▍| 292/311 [00:01<00:00, 348.48it/s, Materializing param=model.layers.26.mlp.up_proj.weight] Loading weights: 94%|█████████▍| 292/311 [00:01<00:00, 348.48it/s, Materializing param=model.layers.26.mlp.up_proj.weight] Loading weights: 88%|████████▊ | 275/311 [00:01<00:00, 345.75it/s, Materializing param=model.layers.24.self_attn.q_norm.weight] Loading weights: 88%|████████▊ | 275/311 [00:01<00:00, 345.75it/s, Materializing param=model.layers.24.self_attn.q_norm.weight] Loading weights: 89%|████████▊ | 276/311 [00:01<00:00, 345.75it/s, Materializing param=model.layers.24.self_attn.q_proj.weight] Loading weights: 89%|████████▊ | 276/311 [00:01<00:00, 345.75it/s, Materializing param=model.layers.24.self_attn.q_proj.weight] Loading weights: 89%|████████▉ | 277/311 [00:01<00:00, 345.75it/s, Materializing param=model.layers.24.self_attn.v_proj.weight] Loading weights: 89%|████████▉ | 277/311 [00:01<00:00, 345.75it/s, Materializing param=model.layers.24.self_attn.v_proj.weight] Loading weights: 89%|████████▉ | 278/311 [00:01<00:00, 345.75it/s, Materializing param=model.layers.25.input_layernorm.weight] Loading weights: 89%|████████▉ | 278/311 [00:01<00:00, 345.75it/s, Materializing param=model.layers.25.input_layernorm.weight] Loading weights: 90%|████████▉ | 279/311 [00:01<00:00, 345.75it/s, Materializing param=model.layers.25.mlp.down_proj.weight] Loading weights: 90%|████████▉ | 279/311 [00:01<00:00, 345.75it/s, Materializing param=model.layers.25.mlp.down_proj.weight] Loading weights: 94%|█████████▍| 293/311 [00:01<00:00, 348.48it/s, Materializing param=model.layers.26.post_attention_layernorm.weight] Loading weights: 94%|█████████▍| 293/311 [00:01<00:00, 348.48it/s, Materializing param=model.layers.26.post_attention_layernorm.weight] Loading weights: 95%|█████████▍| 294/311 [00:01<00:00, 348.48it/s, Materializing param=model.layers.26.self_attn.k_norm.weight] Loading weights: 95%|█████████▍| 294/311 [00:01<00:00, 348.48it/s, Materializing param=model.layers.26.self_attn.k_norm.weight] Loading weights: 95%|█████████▍| 295/311 [00:01<00:00, 348.48it/s, Materializing param=model.layers.26.self_attn.k_proj.weight] Loading weights: 95%|█████████▍| 295/311 [00:01<00:00, 348.48it/s, Materializing param=model.layers.26.self_attn.k_proj.weight] Loading weights: 95%|█████████▌| 296/311 [00:01<00:00, 348.48it/s, Materializing param=model.layers.26.self_attn.o_proj.weight] Loading weights: 95%|█████████▌| 296/311 [00:01<00:00, 348.48it/s, Materializing param=model.layers.26.self_attn.o_proj.weight] Loading weights: 95%|█████████▌| 297/311 [00:01<00:00, 348.48it/s, Materializing param=model.layers.26.self_attn.q_norm.weight] Loading weights: 95%|█████████▌| 297/311 [00:01<00:00, 348.48it/s, Materializing param=model.layers.26.self_attn.q_norm.weight] Loading weights: 90%|█████████ | 280/311 [00:01<00:00, 345.75it/s, Materializing param=model.layers.25.mlp.gate_proj.weight] Loading weights: 96%|█████████▌| 298/311 [00:01<00:00, 348.48it/s, Materializing param=model.layers.26.self_attn.q_proj.weight] Loading weights: 90%|█████████ | 280/311 [00:01<00:00, 345.75it/s, Materializing param=model.layers.25.mlp.gate_proj.weight] Loading weights: 96%|█████████▌| 298/311 [00:01<00:00, 348.48it/s, Materializing param=model.layers.26.self_attn.q_proj.weight] Loading weights: 96%|█████████▌| 299/311 [00:01<00:00, 348.48it/s, Materializing param=model.layers.26.self_attn.v_proj.weight] Loading weights: 96%|█████████▌| 299/311 [00:01<00:00, 348.48it/s, Materializing param=model.layers.26.self_attn.v_proj.weight] Loading weights: 96%|█████████▋| 300/311 [00:01<00:00, 348.48it/s, Materializing param=model.layers.27.input_layernorm.weight] Loading weights: 96%|█████████▋| 300/311 [00:01<00:00, 348.48it/s, Materializing param=model.layers.27.input_layernorm.weight] Loading weights: 97%|█████████▋| 301/311 [00:01<00:00, 348.48it/s, Materializing param=model.layers.27.mlp.down_proj.weight] Loading weights: 97%|█████████▋| 301/311 [00:01<00:00, 348.48it/s, Materializing param=model.layers.27.mlp.down_proj.weight] Loading weights: 90%|█████████ | 281/311 [00:01<00:00, 345.75it/s, Materializing param=model.layers.25.mlp.up_proj.weight] Loading weights: 90%|█████████ | 281/311 [00:01<00:00, 345.75it/s, Materializing param=model.layers.25.mlp.up_proj.weight] Loading weights: 97%|█████████▋| 302/311 [00:01<00:00, 348.48it/s, Materializing param=model.layers.27.mlp.gate_proj.weight] Loading weights: 97%|█████████▋| 302/311 [00:01<00:00, 348.48it/s, Materializing param=model.layers.27.mlp.gate_proj.weight] Loading weights: 91%|█████████ | 282/311 [00:01<00:00, 369.53it/s, Materializing param=model.layers.25.mlp.up_proj.weight] Loading weights: 91%|█████████ | 282/311 [00:01<00:00, 369.53it/s, Materializing param=model.layers.25.post_attention_layernorm.weight] Loading weights: 91%|█████████ | 282/311 [00:01<00:00, 369.53it/s, Materializing param=model.layers.25.post_attention_layernorm.weight] Loading weights: 91%|█████████ | 283/311 [00:01<00:00, 369.53it/s, Materializing param=model.layers.25.self_attn.k_norm.weight] Loading weights: 91%|█████████ | 283/311 [00:01<00:00, 369.53it/s, Materializing param=model.layers.25.self_attn.k_norm.weight] Loading weights: 91%|█████████▏| 284/311 [00:01<00:00, 369.53it/s, Materializing param=model.layers.25.self_attn.k_proj.weight] Loading weights: 91%|█████████▏| 284/311 [00:01<00:00, 369.53it/s, Materializing param=model.layers.25.self_attn.k_proj.weight] Loading weights: 92%|█████████▏| 285/311 [00:01<00:00, 369.53it/s, Materializing param=model.layers.25.self_attn.o_proj.weight] Loading weights: 92%|█████████▏| 285/311 [00:01<00:00, 369.53it/s, Materializing param=model.layers.25.self_attn.o_proj.weight] Loading weights: 92%|█████████▏| 286/311 [00:01<00:00, 369.53it/s, Materializing param=model.layers.25.self_attn.q_norm.weight] Loading weights: 92%|█████████▏| 286/311 [00:01<00:00, 369.53it/s, Materializing param=model.layers.25.self_attn.q_norm.weight] Loading weights: 92%|█████████▏| 287/311 [00:01<00:00, 369.53it/s, Materializing param=model.layers.25.self_attn.q_proj.weight] Loading weights: 92%|█████████▏| 287/311 [00:01<00:00, 369.53it/s, Materializing param=model.layers.25.self_attn.q_proj.weight] Loading weights: 93%|█████████▎| 288/311 [00:01<00:00, 369.53it/s, Materializing param=model.layers.25.self_attn.v_proj.weight] Loading weights: 93%|█████████▎| 288/311 [00:01<00:00, 369.53it/s, Materializing param=model.layers.25.self_attn.v_proj.weight] Loading weights: 93%|█████████▎| 289/311 [00:01<00:00, 369.53it/s, Materializing param=model.layers.26.input_layernorm.weight] Loading weights: 93%|█████████▎| 289/311 [00:01<00:00, 369.53it/s, Materializing param=model.layers.26.input_layernorm.weight] Loading weights: 93%|█████████▎| 290/311 [00:01<00:00, 369.53it/s, Materializing param=model.layers.26.mlp.down_proj.weight] Loading weights: 93%|█████████▎| 290/311 [00:01<00:00, 369.53it/s, Materializing param=model.layers.26.mlp.down_proj.weight] Loading weights: 97%|█████████▋| 303/311 [00:01<00:00, 348.48it/s, Materializing param=model.layers.27.mlp.up_proj.weight] Loading weights: 97%|█████████▋| 303/311 [00:01<00:00, 348.48it/s, Materializing param=model.layers.27.mlp.up_proj.weight] Loading weights: 98%|█████████▊| 304/311 [00:01<00:00, 348.48it/s, Materializing param=model.layers.27.post_attention_layernorm.weight] Loading weights: 98%|█████████▊| 304/311 [00:01<00:00, 348.48it/s, Materializing param=model.layers.27.post_attention_layernorm.weight] Loading weights: 98%|█████████▊| 305/311 [00:01<00:00, 348.48it/s, Materializing param=model.layers.27.self_attn.k_norm.weight] Loading weights: 98%|█████████▊| 305/311 [00:01<00:00, 348.48it/s, Materializing param=model.layers.27.self_attn.k_norm.weight] Loading weights: 98%|█████████▊| 306/311 [00:01<00:00, 348.48it/s, Materializing param=model.layers.27.self_attn.k_proj.weight] Loading weights: 98%|█████████▊| 306/311 [00:01<00:00, 348.48it/s, Materializing param=model.layers.27.self_attn.k_proj.weight] Loading weights: 99%|█████████▊| 307/311 [00:01<00:00, 348.48it/s, Materializing param=model.layers.27.self_attn.o_proj.weight] Loading weights: 99%|█████████▊| 307/311 [00:01<00:00, 348.48it/s, Materializing param=model.layers.27.self_attn.o_proj.weight] Loading weights: 94%|█████████▎| 291/311 [00:01<00:00, 369.53it/s, Materializing param=model.layers.26.mlp.gate_proj.weight] Loading weights: 94%|█████████▎| 291/311 [00:01<00:00, 369.53it/s, Materializing param=model.layers.26.mlp.gate_proj.weight] Loading weights: 99%|█████████▉| 308/311 [00:01<00:00, 348.48it/s, Materializing param=model.layers.27.self_attn.q_norm.weight] Loading weights: 99%|█████████▉| 308/311 [00:01<00:00, 348.48it/s, Materializing param=model.layers.27.self_attn.q_norm.weight] Loading weights: 99%|█████████▉| 309/311 [00:01<00:00, 348.48it/s, Materializing param=model.layers.27.self_attn.q_proj.weight] Loading weights: 99%|█████████▉| 309/311 [00:01<00:00, 348.48it/s, Materializing param=model.layers.27.self_attn.q_proj.weight] Loading weights: 100%|█████████▉| 310/311 [00:01<00:00, 348.48it/s, Materializing param=model.layers.27.self_attn.v_proj.weight] Loading weights: 100%|█████████▉| 310/311 [00:01<00:00, 348.48it/s, Materializing param=model.layers.27.self_attn.v_proj.weight] Loading weights: 100%|██████████| 311/311 [00:01<00:00, 348.48it/s, Materializing param=model.norm.weight] Loading weights: 100%|██████████| 311/311 [00:01<00:00, 348.48it/s, Materializing param=model.norm.weight] Loading weights: 100%|██████████| 311/311 [00:01<00:00, 268.26it/s, Materializing param=model.norm.weight] + Loading weights: 94%|█████████▍| 292/311 [00:01<00:00, 369.53it/s, Materializing param=model.layers.26.mlp.up_proj.weight] Loading weights: 94%|█████████▍| 292/311 [00:01<00:00, 369.53it/s, Materializing param=model.layers.26.mlp.up_proj.weight] Loading weights: 94%|█████████▍| 293/311 [00:01<00:00, 369.53it/s, Materializing param=model.layers.26.post_attention_layernorm.weight] Loading weights: 94%|█████████▍| 293/311 [00:01<00:00, 369.53it/s, Materializing param=model.layers.26.post_attention_layernorm.weight] Loading weights: 95%|█████████▍| 294/311 [00:01<00:00, 369.53it/s, Materializing param=model.layers.26.self_attn.k_norm.weight] Loading weights: 95%|█████████▍| 294/311 [00:01<00:00, 369.53it/s, Materializing param=model.layers.26.self_attn.k_norm.weight] Loading weights: 95%|█████████▍| 295/311 [00:01<00:00, 369.53it/s, Materializing param=model.layers.26.self_attn.k_proj.weight] Loading weights: 95%|█████████▍| 295/311 [00:01<00:00, 369.53it/s, Materializing param=model.layers.26.self_attn.k_proj.weight] Loading weights: 95%|█████████▌| 296/311 [00:01<00:00, 369.53it/s, Materializing param=model.layers.26.self_attn.o_proj.weight] Loading weights: 95%|█████████▌| 296/311 [00:01<00:00, 369.53it/s, Materializing param=model.layers.26.self_attn.o_proj.weight]The tied weights mapping and config for this model specifies to tie model.embed_tokens.weight to lm_head.weight, but both are present in the checkpoints, so we will NOT tie them. You should update the config with `tie_word_embeddings=False` to silence this warning + Loading weights: 95%|█████████▌| 297/311 [00:01<00:00, 369.53it/s, Materializing param=model.layers.26.self_attn.q_norm.weight] Loading weights: 95%|█████████▌| 297/311 [00:01<00:00, 369.53it/s, Materializing param=model.layers.26.self_attn.q_norm.weight] Loading weights: 96%|█████████▌| 298/311 [00:01<00:00, 369.53it/s, Materializing param=model.layers.26.self_attn.q_proj.weight] Loading weights: 96%|█████████▌| 298/311 [00:01<00:00, 369.53it/s, Materializing param=model.layers.26.self_attn.q_proj.weight] Loading weights: 96%|█████████▌| 299/311 [00:01<00:00, 369.53it/s, Materializing param=model.layers.26.self_attn.v_proj.weight] Loading weights: 96%|█████████▌| 299/311 [00:01<00:00, 369.53it/s, Materializing param=model.layers.26.self_attn.v_proj.weight] Loading weights: 96%|█████████▋| 300/311 [00:01<00:00, 369.53it/s, Materializing param=model.layers.27.input_layernorm.weight] Loading weights: 96%|█████████▋| 300/311 [00:01<00:00, 369.53it/s, Materializing param=model.layers.27.input_layernorm.weight] Loading weights: 97%|█████████▋| 301/311 [00:01<00:00, 369.53it/s, Materializing param=model.layers.27.mlp.down_proj.weight] Loading weights: 97%|█████████▋| 301/311 [00:01<00:00, 369.53it/s, Materializing param=model.layers.27.mlp.down_proj.weight] Loading weights: 97%|█████████▋| 302/311 [00:01<00:00, 369.53it/s, Materializing param=model.layers.27.mlp.gate_proj.weight] Loading weights: 97%|█████████▋| 302/311 [00:01<00:00, 369.53it/s, Materializing param=model.layers.27.mlp.gate_proj.weight] Loading weights: 97%|█████████▋| 303/311 [00:01<00:00, 369.53it/s, Materializing param=model.layers.27.mlp.up_proj.weight] Loading weights: 97%|█████████▋| 303/311 [00:01<00:00, 369.53it/s, Materializing param=model.layers.27.mlp.up_proj.weight] Loading weights: 98%|█████████▊| 304/311 [00:01<00:00, 369.53it/s, Materializing param=model.layers.27.post_attention_layernorm.weight] Loading weights: 98%|█████████▊| 304/311 [00:01<00:00, 369.53it/s, Materializing param=model.layers.27.post_attention_layernorm.weight] Loading weights: 98%|█████████▊| 305/311 [00:01<00:00, 369.53it/s, Materializing param=model.layers.27.self_attn.k_norm.weight] Loading weights: 98%|█████████▊| 305/311 [00:01<00:00, 369.53it/s, Materializing param=model.layers.27.self_attn.k_norm.weight] Loading weights: 98%|█████████▊| 306/311 [00:01<00:00, 369.53it/s, Materializing param=model.layers.27.self_attn.k_proj.weight] Loading weights: 98%|█████████▊| 306/311 [00:01<00:00, 369.53it/s, Materializing param=model.layers.27.self_attn.k_proj.weight] Loading weights: 99%|█████████▊| 307/311 [00:01<00:00, 369.53it/s, Materializing param=model.layers.27.self_attn.o_proj.weight] Loading weights: 99%|█████████▊| 307/311 [00:01<00:00, 369.53it/s, Materializing param=model.layers.27.self_attn.o_proj.weight] Loading weights: 99%|█████████▉| 308/311 [00:01<00:00, 369.53it/s, Materializing param=model.layers.27.self_attn.q_norm.weight] Loading weights: 99%|█████████▉| 308/311 [00:01<00:00, 369.53it/s, Materializing param=model.layers.27.self_attn.q_norm.weight] Loading weights: 99%|█████████▉| 309/311 [00:01<00:00, 369.53it/s, Materializing param=model.layers.27.self_attn.q_proj.weight] Loading weights: 99%|█████████▉| 309/311 [00:01<00:00, 369.53it/s, Materializing param=model.layers.27.self_attn.q_proj.weight] Loading weights: 100%|█████████▉| 310/311 [00:01<00:00, 369.53it/s, Materializing param=model.layers.27.self_attn.v_proj.weight] Loading weights: 100%|█████████▉| 310/311 [00:01<00:00, 369.53it/s, Materializing param=model.layers.27.self_attn.v_proj.weight] Loading weights: 100%|██████████| 311/311 [00:01<00:00, 369.53it/s, Materializing param=model.norm.weight] Loading weights: 100%|██████████| 311/311 [00:01<00:00, 369.53it/s, Materializing param=model.norm.weight] Loading weights: 100%|██████████| 311/311 [00:01<00:00, 277.81it/s, Materializing param=model.norm.weight] +The tokenizer has new PAD/BOS/EOS tokens that differ from the model config and generation config. The model config and generation config were aligned accordingly, being updated with the tokenizer's values. Updated tokens: {'bos_token_id': None, 'pad_token_id': 151643}. +The tokenizer has new PAD/BOS/EOS tokens that differ from the model config and generation config. The model config and generation config were aligned accordingly, being updated with the tokenizer's values. Updated tokens: {'bos_token_id': None, 'pad_token_id': 151643}. +The tokenizer has new PAD/BOS/EOS tokens that differ from the model config and generation config. The model config and generation config were aligned accordingly, being updated with the tokenizer's values. Updated tokens: {'bos_token_id': None, 'pad_token_id': 151643}. +The tokenizer has new PAD/BOS/EOS tokens that differ from the model config and generation config. The model config and generation config were aligned accordingly, being updated with the tokenizer's values. Updated tokens: {'bos_token_id': None, 'pad_token_id': 151643}. +The tokenizer has new PAD/BOS/EOS tokens that differ from the model config and generation config. The model config and generation config were aligned accordingly, being updated with the tokenizer's values. Updated tokens: {'bos_token_id': None, 'pad_token_id': 151643}. +The tied weights mapping and config for this model specifies to tie model.embed_tokens.weight to lm_head.weight, but both are present in the checkpoints, so we will NOT tie them. You should update the config with `tie_word_embeddings=False` to silence this warning +ywang29-p4d-debug-1-worker-0:243412:243412 [1] NCCL INFO Comm config Blocking set to 1 +The tied weights mapping and config for this model specifies to tie model.embed_tokens.weight to lm_head.weight, but both are present in the checkpoints, so we will NOT tie them. You should update the config with `tie_word_embeddings=False` to silence this warning +ywang29-p4d-debug-1-worker-0:243416:243416 [5] NCCL INFO Comm config Blocking set to 1 +ywang29-p4d-debug-1-worker-0:243413:243413 [2] NCCL INFO Comm config Blocking set to 1 +ywang29-p4d-debug-1-worker-0:243415:243415 [4] NCCL INFO Comm config Blocking set to 1 +The tokenizer has new PAD/BOS/EOS tokens that differ from the model config and generation config. The model config and generation config were aligned accordingly, being updated with the tokenizer's values. Updated tokens: {'bos_token_id': None, 'pad_token_id': 151643}. +ywang29-p4d-debug-1-worker-0:243414:243414 [3] NCCL INFO Comm config Blocking set to 1 +The tokenizer has new PAD/BOS/EOS tokens that differ from the model config and generation config. The model config and generation config were aligned accordingly, being updated with the tokenizer's values. Updated tokens: {'bos_token_id': None, 'pad_token_id': 151643}. +ywang29-p4d-debug-1-worker-0:243418:243418 [7] NCCL INFO Comm config Blocking set to 1 +ywang29-p4d-debug-1-worker-0:243417:243417 [6] NCCL INFO Comm config Blocking set to 1 +ywang29-p4d-debug-1-worker-0:243417:243417 [6] NCCL INFO Initialized NET plugin Socket +ywang29-p4d-debug-1-worker-0:243418:243418 [7] NCCL INFO Initialized NET plugin Socket +ywang29-p4d-debug-1-worker-0:243415:243415 [4] NCCL INFO Initialized NET plugin Socket +ywang29-p4d-debug-1-worker-0:243416:243416 [5] NCCL INFO Initialized NET plugin Socket +ywang29-p4d-debug-1-worker-0:243413:243413 [2] NCCL INFO Initialized NET plugin Socket +ywang29-p4d-debug-1-worker-0:243412:243412 [1] NCCL INFO Initialized NET plugin Socket +ywang29-p4d-debug-1-worker-0:243414:243414 [3] NCCL INFO Initialized NET plugin Socket +ywang29-p4d-debug-1-worker-0:243417:243417 [6] NCCL INFO Assigned NET plugin Socket to comm +ywang29-p4d-debug-1-worker-0:243417:243417 [6] NCCL INFO Using network Socket +ywang29-p4d-debug-1-worker-0:243411:243411 [0] NCCL INFO Initialized NET plugin Socket +ywang29-p4d-debug-1-worker-0:243418:243418 [7] NCCL INFO Assigned NET plugin Socket to comm +ywang29-p4d-debug-1-worker-0:243418:243418 [7] NCCL INFO Using network Socket +ywang29-p4d-debug-1-worker-0:243415:243415 [4] NCCL INFO Assigned NET plugin Socket to comm +ywang29-p4d-debug-1-worker-0:243412:243412 [1] NCCL INFO Assigned NET plugin Socket to comm +ywang29-p4d-debug-1-worker-0:243415:243415 [4] NCCL INFO Using network Socket +ywang29-p4d-debug-1-worker-0:243413:243413 [2] NCCL INFO Assigned NET plugin Socket to comm +ywang29-p4d-debug-1-worker-0:243412:243412 [1] NCCL INFO Using network Socket +ywang29-p4d-debug-1-worker-0:243414:243414 [3] NCCL INFO Assigned NET plugin Socket to comm +ywang29-p4d-debug-1-worker-0:243416:243416 [5] NCCL INFO Assigned NET plugin Socket to comm +ywang29-p4d-debug-1-worker-0:243413:243413 [2] NCCL INFO Using network Socket +ywang29-p4d-debug-1-worker-0:243414:243414 [3] NCCL INFO Using network Socket +ywang29-p4d-debug-1-worker-0:243416:243416 [5] NCCL INFO Using network Socket +ywang29-p4d-debug-1-worker-0:243411:243411 [0] NCCL INFO Assigned NET plugin Socket to comm +ywang29-p4d-debug-1-worker-0:243411:243411 [0] NCCL INFO Using network Socket +ywang29-p4d-debug-1-worker-0:243418:243418 [7] NCCL INFO ncclCommSplit comm 0x55aba4786d20 rank 7 nranks 8 cudaDev 7 nvmlDev 7 busId a01d0 parent 0x55ab97689c00 splitCount 1 color 1266629538 key 7- Init START +ywang29-p4d-debug-1-worker-0:243413:243413 [2] NCCL INFO ncclCommSplit comm 0x55b7e396d9c0 rank 2 nranks 8 cudaDev 2 nvmlDev 2 busId 201c0 parent 0x55b7d7b1b9d0 splitCount 1 color 1266629538 key 2- Init START +ywang29-p4d-debug-1-worker-0:243412:243412 [1] NCCL INFO ncclCommSplit comm 0x558ee45ba3d0 rank 1 nranks 8 cudaDev 1 nvmlDev 1 busId 101d0 parent 0x558ed75c7da0 splitCount 1 color 1266629538 key 1- Init START +ywang29-p4d-debug-1-worker-0:243415:243415 [4] NCCL INFO ncclCommSplit comm 0x561d4c169060 rank 4 nranks 8 cudaDev 4 nvmlDev 4 busId 901c0 parent 0x561d46f046a0 splitCount 1 color 1266629538 key 4- Init START +ywang29-p4d-debug-1-worker-0:243414:243414 [3] NCCL INFO ncclCommSplit comm 0x55ae95eb9e30 rank 3 nranks 8 cudaDev 3 nvmlDev 3 busId 201d0 parent 0x55ae88a9dd90 splitCount 1 color 1266629538 key 3- Init START +ywang29-p4d-debug-1-worker-0:243417:243417 [6] NCCL INFO ncclCommSplit comm 0x55b500e80270 rank 6 nranks 8 cudaDev 6 nvmlDev 6 busId a01c0 parent 0x55b4f5018ae0 splitCount 1 color 1266629538 key 6- Init START +ywang29-p4d-debug-1-worker-0:243416:243416 [5] NCCL INFO ncclCommSplit comm 0x558fdd7d54e0 rank 5 nranks 8 cudaDev 5 nvmlDev 5 busId 901d0 parent 0x558fd1beff10 splitCount 1 color 1266629538 key 5- Init START +ywang29-p4d-debug-1-worker-0:243411:243411 [0] NCCL INFO ncclCommSplit comm 0x56417bb8e370 rank 0 nranks 8 cudaDev 0 nvmlDev 0 busId 101c0 parent 0x56416eeca870 splitCount 1 color 1266629538 key 0- Init START +ywang29-p4d-debug-1-worker-0:243415:243415 [4] NCCL INFO ncclTopoGetCpuAffinity: Affinity for GPU 4 is 24-47,72-95. (GPU affinity = 24-47,72-95 ; CPU affinity = 0-95). +ywang29-p4d-debug-1-worker-0:243415:243415 [4] NCCL INFO NVLS multicast support is not available on dev 4 (NVLS_NCHANNELS 0) +ywang29-p4d-debug-1-worker-0:243418:243418 [7] NCCL INFO ncclTopoGetCpuAffinity: Affinity for GPU 7 is 24-47,72-95. (GPU affinity = 24-47,72-95 ; CPU affinity = 0-95). +ywang29-p4d-debug-1-worker-0:243418:243418 [7] NCCL INFO NVLS multicast support is not available on dev 7 (NVLS_NCHANNELS 0) +ywang29-p4d-debug-1-worker-0:243411:243411 [0] NCCL INFO ncclTopoGetCpuAffinity: Affinity for GPU 0 is 0-23,48-71. (GPU affinity = 0-23,48-71 ; CPU affinity = 0-95). +ywang29-p4d-debug-1-worker-0:243411:243411 [0] NCCL INFO NVLS multicast support is not available on dev 0 (NVLS_NCHANNELS 0) +ywang29-p4d-debug-1-worker-0:243416:243416 [5] NCCL INFO ncclTopoGetCpuAffinity: Affinity for GPU 5 is 24-47,72-95. (GPU affinity = 24-47,72-95 ; CPU affinity = 0-95). +ywang29-p4d-debug-1-worker-0:243416:243416 [5] NCCL INFO NVLS multicast support is not available on dev 5 (NVLS_NCHANNELS 0) +ywang29-p4d-debug-1-worker-0:243413:243413 [2] NCCL INFO ncclTopoGetCpuAffinity: Affinity for GPU 2 is 0-23,48-71. (GPU affinity = 0-23,48-71 ; CPU affinity = 0-95). +ywang29-p4d-debug-1-worker-0:243413:243413 [2] NCCL INFO NVLS multicast support is not available on dev 2 (NVLS_NCHANNELS 0) +ywang29-p4d-debug-1-worker-0:243414:243414 [3] NCCL INFO ncclTopoGetCpuAffinity: Affinity for GPU 3 is 0-23,48-71. (GPU affinity = 0-23,48-71 ; CPU affinity = 0-95). +ywang29-p4d-debug-1-worker-0:243414:243414 [3] NCCL INFO NVLS multicast support is not available on dev 3 (NVLS_NCHANNELS 0) +ywang29-p4d-debug-1-worker-0:243412:243412 [1] NCCL INFO ncclTopoGetCpuAffinity: Affinity for GPU 1 is 0-23,48-71. (GPU affinity = 0-23,48-71 ; CPU affinity = 0-95). +ywang29-p4d-debug-1-worker-0:243412:243412 [1] NCCL INFO NVLS multicast support is not available on dev 1 (NVLS_NCHANNELS 0) +ywang29-p4d-debug-1-worker-0:243417:243417 [6] NCCL INFO ncclTopoGetCpuAffinity: Affinity for GPU 6 is 24-47,72-95. (GPU affinity = 24-47,72-95 ; CPU affinity = 0-95). +ywang29-p4d-debug-1-worker-0:243417:243417 [6] NCCL INFO NVLS multicast support is not available on dev 6 (NVLS_NCHANNELS 0) +ywang29-p4d-debug-1-worker-0:243416:243416 [5] NCCL INFO comm 0x558fdd7d54e0 rank 5 nRanks 8 nNodes 1 localRanks 8 localRank 5 MNNVL 0 +ywang29-p4d-debug-1-worker-0:243414:243414 [3] NCCL INFO comm 0x55ae95eb9e30 rank 3 nRanks 8 nNodes 1 localRanks 8 localRank 3 MNNVL 0 +ywang29-p4d-debug-1-worker-0:243417:243417 [6] NCCL INFO comm 0x55b500e80270 rank 6 nRanks 8 nNodes 1 localRanks 8 localRank 6 MNNVL 0 +ywang29-p4d-debug-1-worker-0:243415:243415 [4] NCCL INFO comm 0x561d4c169060 rank 4 nRanks 8 nNodes 1 localRanks 8 localRank 4 MNNVL 0 +ywang29-p4d-debug-1-worker-0:243412:243412 [1] NCCL INFO comm 0x558ee45ba3d0 rank 1 nRanks 8 nNodes 1 localRanks 8 localRank 1 MNNVL 0 +ywang29-p4d-debug-1-worker-0:243413:243413 [2] NCCL INFO comm 0x55b7e396d9c0 rank 2 nRanks 8 nNodes 1 localRanks 8 localRank 2 MNNVL 0 +ywang29-p4d-debug-1-worker-0:243418:243418 [7] NCCL INFO comm 0x55aba4786d20 rank 7 nRanks 8 nNodes 1 localRanks 8 localRank 7 MNNVL 0 +ywang29-p4d-debug-1-worker-0:243411:243411 [0] NCCL INFO comm 0x56417bb8e370 rank 0 nRanks 8 nNodes 1 localRanks 8 localRank 0 MNNVL 0 +ywang29-p4d-debug-1-worker-0:243411:243411 [0] NCCL INFO Channel 00/24 : 0 1 2 3 4 5 6 7 +ywang29-p4d-debug-1-worker-0:243417:243417 [6] NCCL INFO Trees [0] 7/-1/-1->6->5 [1] 7/-1/-1->6->5 [2] 7/-1/-1->6->5 [3] 7/-1/-1->6->5 [4] 7/-1/-1->6->5 [5] 7/-1/-1->6->5 [6] 7/-1/-1->6->5 [7] 7/-1/-1->6->5 [8] 7/-1/-1->6->5 [9] 7/-1/-1->6->5 [10] 7/-1/-1->6->5 [11] 7/-1/-1->6->5 [12] 7/-1/-1->6->5 [13] 7/-1/-1->6->5 [14] 7/-1/-1->6->5 [15] 7/-1/-1->6->5 [16] 7/-1/-1->6->5 [17] 7/-1/-1->6->5 [18] 7/-1/-1->6->5 [19] 7/-1/-1->6->5 [20] 7/-1/-1->6->5 [21] 7/-1/-1->6->5 [22] 7/-1/-1->6->5 [23] 7/-1/-1->6->5 +ywang29-p4d-debug-1-worker-0:243417:243417 [6] NCCL INFO P2P Chunksize set to 524288 +ywang29-p4d-debug-1-worker-0:243416:243416 [5] NCCL INFO Trees [0] 6/-1/-1->5->4 [1] 6/-1/-1->5->4 [2] 6/-1/-1->5->4 [3] 6/-1/-1->5->4 [4] 6/-1/-1->5->4 [5] 6/-1/-1->5->4 [6] 6/-1/-1->5->4 [7] 6/-1/-1->5->4 [8] 6/-1/-1->5->4 [9] 6/-1/-1->5->4 [10] 6/-1/-1->5->4 [11] 6/-1/-1->5->4 [12] 6/-1/-1->5->4 [13] 6/-1/-1->5->4 [14] 6/-1/-1->5->4 [15] 6/-1/-1->5->4 [16] 6/-1/-1->5->4 [17] 6/-1/-1->5->4 [18] 6/-1/-1->5->4 [19] 6/-1/-1->5->4 [20] 6/-1/-1->5->4 [21] 6/-1/-1->5->4 [22] 6/-1/-1->5->4 [23] 6/-1/-1->5->4 +ywang29-p4d-debug-1-worker-0:243414:243414 [3] NCCL INFO Trees [0] 4/-1/-1->3->2 [1] 4/-1/-1->3->2 [2] 4/-1/-1->3->2 [3] 4/-1/-1->3->2 [4] 4/-1/-1->3->2 [5] 4/-1/-1->3->2 [6] 4/-1/-1->3->2 [7] 4/-1/-1->3->2 [8] 4/-1/-1->3->2 [9] 4/-1/-1->3->2 [10] 4/-1/-1->3->2 [11] 4/-1/-1->3->2 [12] 4/-1/-1->3->2 [13] 4/-1/-1->3->2 [14] 4/-1/-1->3->2 [15] 4/-1/-1->3->2 [16] 4/-1/-1->3->2 [17] 4/-1/-1->3->2 [18] 4/-1/-1->3->2 [19] 4/-1/-1->3->2 [20] 4/-1/-1->3->2 [21] 4/-1/-1->3->2 [22] 4/-1/-1->3->2 [23] 4/-1/-1->3->2 +ywang29-p4d-debug-1-worker-0:243411:243411 [0] NCCL INFO Channel 01/24 : 0 1 2 3 4 5 6 7 +ywang29-p4d-debug-1-worker-0:243416:243416 [5] NCCL INFO P2P Chunksize set to 524288 +ywang29-p4d-debug-1-worker-0:243412:243412 [1] NCCL INFO Trees [0] 2/-1/-1->1->0 [1] 2/-1/-1->1->0 [2] 2/-1/-1->1->0 [3] 2/-1/-1->1->0 [4] 2/-1/-1->1->0 [5] 2/-1/-1->1->0 [6] 2/-1/-1->1->0 [7] 2/-1/-1->1->0 [8] 2/-1/-1->1->0 [9] 2/-1/-1->1->0 [10] 2/-1/-1->1->0 [11] 2/-1/-1->1->0 [12] 2/-1/-1->1->0 [13] 2/-1/-1->1->0 [14] 2/-1/-1->1->0 [15] 2/-1/-1->1->0 [16] 2/-1/-1->1->0 [17] 2/-1/-1->1->0 [18] 2/-1/-1->1->0 [19] 2/-1/-1->1->0 [20] 2/-1/-1->1->0 [21] 2/-1/-1->1->0 [22] 2/-1/-1->1->0 [23] 2/-1/-1->1->0 +ywang29-p4d-debug-1-worker-0:243411:243411 [0] NCCL INFO Channel 02/24 : 0 1 2 3 4 5 6 7 +ywang29-p4d-debug-1-worker-0:243414:243414 [3] NCCL INFO P2P Chunksize set to 524288 +ywang29-p4d-debug-1-worker-0:243413:243413 [2] NCCL INFO Trees [0] 3/-1/-1->2->1 [1] 3/-1/-1->2->1 [2] 3/-1/-1->2->1 [3] 3/-1/-1->2->1 [4] 3/-1/-1->2->1 [5] 3/-1/-1->2->1 [6] 3/-1/-1->2->1 [7] 3/-1/-1->2->1 [8] 3/-1/-1->2->1 [9] 3/-1/-1->2->1 [10] 3/-1/-1->2->1 [11] 3/-1/-1->2->1 [12] 3/-1/-1->2->1 [13] 3/-1/-1->2->1 [14] 3/-1/-1->2->1 [15] 3/-1/-1->2->1 [16] 3/-1/-1->2->1 [17] 3/-1/-1->2->1 [18] 3/-1/-1->2->1 [19] 3/-1/-1->2->1 [20] 3/-1/-1->2->1 [21] 3/-1/-1->2->1 [22] 3/-1/-1->2->1 [23] 3/-1/-1->2->1 +ywang29-p4d-debug-1-worker-0:243418:243418 [7] NCCL INFO Trees [0] -1/-1/-1->7->6 [1] -1/-1/-1->7->6 [2] -1/-1/-1->7->6 [3] -1/-1/-1->7->6 [4] -1/-1/-1->7->6 [5] -1/-1/-1->7->6 [6] -1/-1/-1->7->6 [7] -1/-1/-1->7->6 [8] -1/-1/-1->7->6 [9] -1/-1/-1->7->6 [10] -1/-1/-1->7->6 [11] -1/-1/-1->7->6 [12] -1/-1/-1->7->6 [13] -1/-1/-1->7->6 [14] -1/-1/-1->7->6 [15] -1/-1/-1->7->6 [16] -1/-1/-1->7->6 [17] -1/-1/-1->7->6 [18] -1/-1/-1->7->6 [19] -1/-1/-1->7->6 [20] -1/-1/-1->7->6 [21] -1/-1/-1->7->6 [22] -1/-1/-1->7->6 [23] -1/-1/-1->7->6 +ywang29-p4d-debug-1-worker-0:243415:243415 [4] NCCL INFO Trees [0] 5/-1/-1->4->3 [1] 5/-1/-1->4->3 [2] 5/-1/-1->4->3 [3] 5/-1/-1->4->3 [4] 5/-1/-1->4->3 [5] 5/-1/-1->4->3 [6] 5/-1/-1->4->3 [7] 5/-1/-1->4->3 [8] 5/-1/-1->4->3 [9] 5/-1/-1->4->3 [10] 5/-1/-1->4->3 [11] 5/-1/-1->4->3 [12] 5/-1/-1->4->3 [13] 5/-1/-1->4->3 [14] 5/-1/-1->4->3 [15] 5/-1/-1->4->3 [16] 5/-1/-1->4->3 [17] 5/-1/-1->4->3 [18] 5/-1/-1->4->3 [19] 5/-1/-1->4->3 [20] 5/-1/-1->4->3 [21] 5/-1/-1->4->3 [22] 5/-1/-1->4->3 [23] 5/-1/-1->4->3 +ywang29-p4d-debug-1-worker-0:243412:243412 [1] NCCL INFO P2P Chunksize set to 524288 +ywang29-p4d-debug-1-worker-0:243413:243413 [2] NCCL INFO P2P Chunksize set to 524288 +ywang29-p4d-debug-1-worker-0:243418:243418 [7] NCCL INFO P2P Chunksize set to 524288 +ywang29-p4d-debug-1-worker-0:243411:243411 [0] NCCL INFO Channel 03/24 : 0 1 2 3 4 5 6 7 +ywang29-p4d-debug-1-worker-0:243415:243415 [4] NCCL INFO P2P Chunksize set to 524288 +ywang29-p4d-debug-1-worker-0:243411:243411 [0] NCCL INFO Channel 04/24 : 0 1 2 3 4 5 6 7 +ywang29-p4d-debug-1-worker-0:243411:243411 [0] NCCL INFO Channel 05/24 : 0 1 2 3 4 5 6 7 +ywang29-p4d-debug-1-worker-0:243411:243411 [0] NCCL INFO Channel 06/24 : 0 1 2 3 4 5 6 7 +ywang29-p4d-debug-1-worker-0:243411:243411 [0] NCCL INFO Channel 07/24 : 0 1 2 3 4 5 6 7 +ywang29-p4d-debug-1-worker-0:243411:243411 [0] NCCL INFO Channel 08/24 : 0 1 2 3 4 5 6 7 +ywang29-p4d-debug-1-worker-0:243411:243411 [0] NCCL INFO Channel 09/24 : 0 1 2 3 4 5 6 7 +ywang29-p4d-debug-1-worker-0:243411:243411 [0] NCCL INFO Channel 10/24 : 0 1 2 3 4 5 6 7 +ywang29-p4d-debug-1-worker-0:243411:243411 [0] NCCL INFO Channel 11/24 : 0 1 2 3 4 5 6 7 +ywang29-p4d-debug-1-worker-0:243411:243411 [0] NCCL INFO Channel 12/24 : 0 1 2 3 4 5 6 7 +ywang29-p4d-debug-1-worker-0:243411:243411 [0] NCCL INFO Channel 13/24 : 0 1 2 3 4 5 6 7 +ywang29-p4d-debug-1-worker-0:243411:243411 [0] NCCL INFO Channel 14/24 : 0 1 2 3 4 5 6 7 +ywang29-p4d-debug-1-worker-0:243411:243411 [0] NCCL INFO Channel 15/24 : 0 1 2 3 4 5 6 7 +ywang29-p4d-debug-1-worker-0:243411:243411 [0] NCCL INFO Channel 16/24 : 0 1 2 3 4 5 6 7 +ywang29-p4d-debug-1-worker-0:243411:243411 [0] NCCL INFO Channel 17/24 : 0 1 2 3 4 5 6 7 +ywang29-p4d-debug-1-worker-0:243411:243411 [0] NCCL INFO Channel 18/24 : 0 1 2 3 4 5 6 7 +ywang29-p4d-debug-1-worker-0:243411:243411 [0] NCCL INFO Channel 19/24 : 0 1 2 3 4 5 6 7 +ywang29-p4d-debug-1-worker-0:243411:243411 [0] NCCL INFO Channel 20/24 : 0 1 2 3 4 5 6 7 +ywang29-p4d-debug-1-worker-0:243411:243411 [0] NCCL INFO Channel 21/24 : 0 1 2 3 4 5 6 7 +ywang29-p4d-debug-1-worker-0:243411:243411 [0] NCCL INFO Channel 22/24 : 0 1 2 3 4 5 6 7 +ywang29-p4d-debug-1-worker-0:243411:243411 [0] NCCL INFO Channel 23/24 : 0 1 2 3 4 5 6 7 +ywang29-p4d-debug-1-worker-0:243411:243411 [0] NCCL INFO Trees [0] 1/-1/-1->0->-1 [1] 1/-1/-1->0->-1 [2] 1/-1/-1->0->-1 [3] 1/-1/-1->0->-1 [4] 1/-1/-1->0->-1 [5] 1/-1/-1->0->-1 [6] 1/-1/-1->0->-1 [7] 1/-1/-1->0->-1 [8] 1/-1/-1->0->-1 [9] 1/-1/-1->0->-1 [10] 1/-1/-1->0->-1 [11] 1/-1/-1->0->-1 [12] 1/-1/-1->0->-1 [13] 1/-1/-1->0->-1 [14] 1/-1/-1->0->-1 [15] 1/-1/-1->0->-1 [16] 1/-1/-1->0->-1 [17] 1/-1/-1->0->-1 [18] 1/-1/-1->0->-1 [19] 1/-1/-1->0->-1 [20] 1/-1/-1->0->-1 [21] 1/-1/-1->0->-1 [22] 1/-1/-1->0->-1 [23] 1/-1/-1->0->-1 +ywang29-p4d-debug-1-worker-0:243411:243411 [0] NCCL INFO P2P Chunksize set to 524288 +ywang29-p4d-debug-1-worker-0:243412:243412 [1] NCCL INFO Check P2P Type isAllDirectP2p 1 directMode 0 isAllCudaP2p 1 +ywang29-p4d-debug-1-worker-0:243416:243416 [5] NCCL INFO Check P2P Type isAllDirectP2p 1 directMode 0 isAllCudaP2p 1 +ywang29-p4d-debug-1-worker-0:243412:244109 [0] NCCL INFO [Proxy Service] Device 1 CPU core 0 +ywang29-p4d-debug-1-worker-0:243412:244110 [0] NCCL INFO [Proxy Service UDS] Device 1 CPU core 49 +ywang29-p4d-debug-1-worker-0:243416:244111 [0] NCCL INFO [Proxy Service] Device 5 CPU core 30 +ywang29-p4d-debug-1-worker-0:243416:244112 [0] NCCL INFO [Proxy Service UDS] Device 5 CPU core 31 +ywang29-p4d-debug-1-worker-0:243418:243418 [7] NCCL INFO Check P2P Type isAllDirectP2p 1 directMode 0 isAllCudaP2p 1 +ywang29-p4d-debug-1-worker-0:243418:244113 [0] NCCL INFO [Proxy Service] Device 7 CPU core 80 +ywang29-p4d-debug-1-worker-0:243418:244114 [0] NCCL INFO [Proxy Service UDS] Device 7 CPU core 33 +ywang29-p4d-debug-1-worker-0:243413:243413 [2] NCCL INFO Check P2P Type isAllDirectP2p 1 directMode 0 isAllCudaP2p 1 +ywang29-p4d-debug-1-worker-0:243413:244115 [0] NCCL INFO [Proxy Service] Device 2 CPU core 52 +ywang29-p4d-debug-1-worker-0:243413:244116 [0] NCCL INFO [Proxy Service UDS] Device 2 CPU core 8 +ywang29-p4d-debug-1-worker-0:243417:243417 [6] NCCL INFO Check P2P Type isAllDirectP2p 1 directMode 0 isAllCudaP2p 1 +ywang29-p4d-debug-1-worker-0:243417:244117 [0] NCCL INFO [Proxy Service] Device 6 CPU core 87 +ywang29-p4d-debug-1-worker-0:243417:244118 [0] NCCL INFO [Proxy Service UDS] Device 6 CPU core 43 +ywang29-p4d-debug-1-worker-0:243414:243414 [3] NCCL INFO Check P2P Type isAllDirectP2p 1 directMode 0 isAllCudaP2p 1 +ywang29-p4d-debug-1-worker-0:243414:244119 [0] NCCL INFO [Proxy Service] Device 3 CPU core 65 +ywang29-p4d-debug-1-worker-0:243414:244120 [0] NCCL INFO [Proxy Service UDS] Device 3 CPU core 66 +ywang29-p4d-debug-1-worker-0:243415:243415 [4] NCCL INFO Check P2P Type isAllDirectP2p 1 directMode 0 isAllCudaP2p 1 +ywang29-p4d-debug-1-worker-0:243415:244122 [0] NCCL INFO [Proxy Service UDS] Device 4 CPU core 82 +ywang29-p4d-debug-1-worker-0:243415:244121 [0] NCCL INFO [Proxy Service] Device 4 CPU core 37 +ywang29-p4d-debug-1-worker-0:243411:243411 [0] NCCL INFO Check P2P Type isAllDirectP2p 1 directMode 0 isAllCudaP2p 1 +ywang29-p4d-debug-1-worker-0:243411:244123 [0] NCCL INFO [Proxy Service] Device 0 CPU core 11 +ywang29-p4d-debug-1-worker-0:243411:244124 [0] NCCL INFO [Proxy Service UDS] Device 0 CPU core 61 +ywang29-p4d-debug-1-worker-0:243412:243412 [1] NCCL INFO NET/OFI NCCL_OFI_TUNER is not available for platform : p4de.24xlarge, Fall back to NCCL's tuner +ywang29-p4d-debug-1-worker-0:243412:243412 [1] NCCL INFO threadThresholds 8/8/64 | 64/8/64 | 512 | 512 +ywang29-p4d-debug-1-worker-0:243412:243412 [1] NCCL INFO 24 coll channels, 24 collnet channels, 0 nvls channels, 32 p2p channels, 32 p2p channels per peer +ywang29-p4d-debug-1-worker-0:243415:243415 [4] NCCL INFO NET/OFI NCCL_OFI_TUNER is not available for platform : p4de.24xlarge, Fall back to NCCL's tuner +ywang29-p4d-debug-1-worker-0:243415:243415 [4] NCCL INFO threadThresholds 8/8/64 | 64/8/64 | 512 | 512 +ywang29-p4d-debug-1-worker-0:243415:243415 [4] NCCL INFO 24 coll channels, 24 collnet channels, 0 nvls channels, 32 p2p channels, 32 p2p channels per peer +ywang29-p4d-debug-1-worker-0:243417:243417 [6] NCCL INFO NET/OFI NCCL_OFI_TUNER is not available for platform : p4de.24xlarge, Fall back to NCCL's tuner +ywang29-p4d-debug-1-worker-0:243417:243417 [6] NCCL INFO threadThresholds 8/8/64 | 64/8/64 | 512 | 512 +ywang29-p4d-debug-1-worker-0:243417:243417 [6] NCCL INFO 24 coll channels, 24 collnet channels, 0 nvls channels, 32 p2p channels, 32 p2p channels per peer +ywang29-p4d-debug-1-worker-0:243414:243414 [3] NCCL INFO NET/OFI NCCL_OFI_TUNER is not available for platform : p4de.24xlarge, Fall back to NCCL's tuner +ywang29-p4d-debug-1-worker-0:243414:243414 [3] NCCL INFO threadThresholds 8/8/64 | 64/8/64 | 512 | 512 +ywang29-p4d-debug-1-worker-0:243414:243414 [3] NCCL INFO 24 coll channels, 24 collnet channels, 0 nvls channels, 32 p2p channels, 32 p2p channels per peer +ywang29-p4d-debug-1-worker-0:243413:243413 [2] NCCL INFO NET/OFI NCCL_OFI_TUNER is not available for platform : p4de.24xlarge, Fall back to NCCL's tuner +ywang29-p4d-debug-1-worker-0:243413:243413 [2] NCCL INFO threadThresholds 8/8/64 | 64/8/64 | 512 | 512 +ywang29-p4d-debug-1-worker-0:243413:243413 [2] NCCL INFO 24 coll channels, 24 collnet channels, 0 nvls channels, 32 p2p channels, 32 p2p channels per peer +ywang29-p4d-debug-1-worker-0:243416:243416 [5] NCCL INFO NET/OFI NCCL_OFI_TUNER is not available for platform : p4de.24xlarge, Fall back to NCCL's tuner +ywang29-p4d-debug-1-worker-0:243416:243416 [5] NCCL INFO threadThresholds 8/8/64 | 64/8/64 | 512 | 512 +ywang29-p4d-debug-1-worker-0:243416:243416 [5] NCCL INFO 24 coll channels, 24 collnet channels, 0 nvls channels, 32 p2p channels, 32 p2p channels per peer +ywang29-p4d-debug-1-worker-0:243418:243418 [7] NCCL INFO NET/OFI NCCL_OFI_TUNER is not available for platform : p4de.24xlarge, Fall back to NCCL's tuner +ywang29-p4d-debug-1-worker-0:243418:243418 [7] NCCL INFO threadThresholds 8/8/64 | 64/8/64 | 512 | 512 +ywang29-p4d-debug-1-worker-0:243418:243418 [7] NCCL INFO 24 coll channels, 24 collnet channels, 0 nvls channels, 32 p2p channels, 32 p2p channels per peer +ywang29-p4d-debug-1-worker-0:243411:243411 [0] NCCL INFO NET/OFI NCCL_OFI_TUNER is not available for platform : p4de.24xlarge, Fall back to NCCL's tuner +ywang29-p4d-debug-1-worker-0:243411:243411 [0] NCCL INFO threadThresholds 8/8/64 | 64/8/64 | 512 | 512 +ywang29-p4d-debug-1-worker-0:243411:243411 [0] NCCL INFO 24 coll channels, 24 collnet channels, 0 nvls channels, 32 p2p channels, 32 p2p channels per peer +ywang29-p4d-debug-1-worker-0:243411:243411 [0] NCCL INFO CC Off, workFifoBytes 1048576 +ywang29-p4d-debug-1-worker-0:243418:243418 [7] NCCL INFO ncclCommSplit comm 0x55aba4786d20 rank 7 nranks 8 cudaDev 7 nvmlDev 7 busId a01d0 parent 0x55ab97689c00 splitCount 1 color 1266629538 key 7 - Init COMPLETE +ywang29-p4d-debug-1-worker-0:243414:243414 [3] NCCL INFO ncclCommSplit comm 0x55ae95eb9e30 rank 3 nranks 8 cudaDev 3 nvmlDev 3 busId 201d0 parent 0x55ae88a9dd90 splitCount 1 color 1266629538 key 3 - Init COMPLETE +ywang29-p4d-debug-1-worker-0:243412:243412 [1] NCCL INFO ncclCommSplit comm 0x558ee45ba3d0 rank 1 nranks 8 cudaDev 1 nvmlDev 1 busId 101d0 parent 0x558ed75c7da0 splitCount 1 color 1266629538 key 1 - Init COMPLETE +ywang29-p4d-debug-1-worker-0:243416:243416 [5] NCCL INFO ncclCommSplit comm 0x558fdd7d54e0 rank 5 nranks 8 cudaDev 5 nvmlDev 5 busId 901d0 parent 0x558fd1beff10 splitCount 1 color 1266629538 key 5 - Init COMPLETE +ywang29-p4d-debug-1-worker-0:243418:243418 [7] NCCL INFO Init timings - ncclCommSplit: rank 7 nranks 8 total 0.17 (kernels 0.00, alloc 0.00, bootstrap 0.00, allgathers 0.01, topo 0.03, graphs 0.00, connections 0.06, rest 0.07) +ywang29-p4d-debug-1-worker-0:243412:243412 [1] NCCL INFO Init timings - ncclCommSplit: rank 1 nranks 8 total 0.89 (kernels 0.00, alloc 0.00, bootstrap 0.00, allgathers 0.00, topo 0.03, graphs 0.00, connections 0.06, rest 0.79) +ywang29-p4d-debug-1-worker-0:243414:243414 [3] NCCL INFO Init timings - ncclCommSplit: rank 3 nranks 8 total 0.58 (kernels 0.00, alloc 0.00, bootstrap 0.00, allgathers 0.01, topo 0.03, graphs 0.00, connections 0.06, rest 0.48) +ywang29-p4d-debug-1-worker-0:243416:243416 [5] NCCL INFO Init timings - ncclCommSplit: rank 5 nranks 8 total 0.78 (kernels 0.00, alloc 0.00, bootstrap 0.00, allgathers 0.01, topo 0.03, graphs 0.00, connections 0.06, rest 0.68) +ywang29-p4d-debug-1-worker-0:243411:243411 [0] NCCL INFO ncclCommSplit comm 0x56417bb8e370 rank 0 nranks 8 cudaDev 0 nvmlDev 0 busId 101c0 parent 0x56416eeca870 splitCount 1 color 1266629538 key 0 - Init COMPLETE +ywang29-p4d-debug-1-worker-0:243415:243415 [4] NCCL INFO ncclCommSplit comm 0x561d4c169060 rank 4 nranks 8 cudaDev 4 nvmlDev 4 busId 901c0 parent 0x561d46f046a0 splitCount 1 color 1266629538 key 4 - Init COMPLETE +ywang29-p4d-debug-1-worker-0:243411:243411 [0] NCCL INFO Init timings - ncclCommSplit: rank 0 nranks 8 total 1.50 (kernels 0.00, alloc 0.00, bootstrap 0.00, allgathers 0.01, topo 0.03, graphs 0.00, connections 0.06, rest 1.41) +ywang29-p4d-debug-1-worker-0:243413:243413 [2] NCCL INFO ncclCommSplit comm 0x55b7e396d9c0 rank 2 nranks 8 cudaDev 2 nvmlDev 2 busId 201c0 parent 0x55b7d7b1b9d0 splitCount 1 color 1266629538 key 2 - Init COMPLETE +ywang29-p4d-debug-1-worker-0:243417:243417 [6] NCCL INFO ncclCommSplit comm 0x55b500e80270 rank 6 nranks 8 cudaDev 6 nvmlDev 6 busId a01c0 parent 0x55b4f5018ae0 splitCount 1 color 1266629538 key 6 - Init COMPLETE +ywang29-p4d-debug-1-worker-0:243415:243415 [4] NCCL INFO Init timings - ncclCommSplit: rank 4 nranks 8 total 0.65 (kernels 0.00, alloc 0.00, bootstrap 0.00, allgathers 0.01, topo 0.03, graphs 0.00, connections 0.06, rest 0.55) +ywang29-p4d-debug-1-worker-0:243413:243413 [2] NCCL INFO Init timings - ncclCommSplit: rank 2 nranks 8 total 0.66 (kernels 0.00, alloc 0.00, bootstrap 0.00, allgathers 0.01, topo 0.03, graphs 0.00, connections 0.06, rest 0.56) +ywang29-p4d-debug-1-worker-0:243417:243417 [6] NCCL INFO Init timings - ncclCommSplit: rank 6 nranks 8 total 0.12 (kernels 0.00, alloc 0.00, bootstrap 0.00, allgathers 0.00, topo 0.04, graphs 0.00, connections 0.06, rest 0.02) +ywang29-p4d-debug-1-worker-0:243414:243414 [3] NCCL INFO Channel 00/0 : 3[3] -> 4[4] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243414:243414 [3] NCCL INFO Channel 01/0 : 3[3] -> 4[4] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243414:243414 [3] NCCL INFO Channel 02/0 : 3[3] -> 4[4] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243414:243414 [3] NCCL INFO Channel 03/0 : 3[3] -> 4[4] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243413:243413 [2] NCCL INFO Channel 00/0 : 2[2] -> 3[3] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243411:243411 [0] NCCL INFO Channel 00/0 : 0[0] -> 1[1] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243417:243417 [6] NCCL INFO Channel 00/0 : 6[6] -> 7[7] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243414:243414 [3] NCCL INFO Channel 04/0 : 3[3] -> 4[4] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243413:243413 [2] NCCL INFO Channel 01/0 : 2[2] -> 3[3] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243413:243413 [2] NCCL INFO Channel 02/0 : 2[2] -> 3[3] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243411:243411 [0] NCCL INFO Channel 01/0 : 0[0] -> 1[1] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243417:243417 [6] NCCL INFO Channel 01/0 : 6[6] -> 7[7] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243412:243412 [1] NCCL INFO Channel 00/0 : 1[1] -> 2[2] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243415:243415 [4] NCCL INFO Channel 00/0 : 4[4] -> 5[5] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243416:243416 [5] NCCL INFO Channel 00/0 : 5[5] -> 6[6] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243416:243416 [5] NCCL INFO Channel 01/0 : 5[5] -> 6[6] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243413:243413 [2] NCCL INFO Channel 03/0 : 2[2] -> 3[3] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243415:243415 [4] NCCL INFO Channel 01/0 : 4[4] -> 5[5] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243412:243412 [1] NCCL INFO Channel 01/0 : 1[1] -> 2[2] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243414:243414 [3] NCCL INFO Channel 05/0 : 3[3] -> 4[4] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243418:243418 [7] NCCL INFO Channel 00/0 : 7[7] -> 0[0] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243411:243411 [0] NCCL INFO Channel 02/0 : 0[0] -> 1[1] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243417:243417 [6] NCCL INFO Channel 02/0 : 6[6] -> 7[7] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243417:243417 [6] NCCL INFO Channel 03/0 : 6[6] -> 7[7] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243413:243413 [2] NCCL INFO Channel 04/0 : 2[2] -> 3[3] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243416:243416 [5] NCCL INFO Channel 02/0 : 5[5] -> 6[6] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243415:243415 [4] NCCL INFO Channel 02/0 : 4[4] -> 5[5] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243412:243412 [1] NCCL INFO Channel 02/0 : 1[1] -> 2[2] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243418:243418 [7] NCCL INFO Channel 01/0 : 7[7] -> 0[0] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243414:243414 [3] NCCL INFO Channel 06/0 : 3[3] -> 4[4] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243414:243414 [3] NCCL INFO Channel 07/0 : 3[3] -> 4[4] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243417:243417 [6] NCCL INFO Channel 04/0 : 6[6] -> 7[7] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243411:243411 [0] NCCL INFO Channel 03/0 : 0[0] -> 1[1] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243415:243415 [4] NCCL INFO Channel 03/0 : 4[4] -> 5[5] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243414:243414 [3] NCCL INFO Channel 08/0 : 3[3] -> 4[4] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243418:243418 [7] NCCL INFO Channel 02/0 : 7[7] -> 0[0] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243413:243413 [2] NCCL INFO Channel 05/0 : 2[2] -> 3[3] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243412:243412 [1] NCCL INFO Channel 03/0 : 1[1] -> 2[2] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243412:243412 [1] NCCL INFO Channel 04/0 : 1[1] -> 2[2] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243418:243418 [7] NCCL INFO Channel 03/0 : 7[7] -> 0[0] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243412:243412 [1] NCCL INFO Channel 05/0 : 1[1] -> 2[2] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243411:243411 [0] NCCL INFO Channel 04/0 : 0[0] -> 1[1] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243417:243417 [6] NCCL INFO Channel 05/0 : 6[6] -> 7[7] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243414:243414 [3] NCCL INFO Channel 09/0 : 3[3] -> 4[4] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243416:243416 [5] NCCL INFO Channel 03/0 : 5[5] -> 6[6] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243413:243413 [2] NCCL INFO Channel 06/0 : 2[2] -> 3[3] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243415:243415 [4] NCCL INFO Channel 04/0 : 4[4] -> 5[5] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243413:243413 [2] NCCL INFO Channel 07/0 : 2[2] -> 3[3] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243417:243417 [6] NCCL INFO Channel 06/0 : 6[6] -> 7[7] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243412:243412 [1] NCCL INFO Channel 06/0 : 1[1] -> 2[2] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243418:243418 [7] NCCL INFO Channel 04/0 : 7[7] -> 0[0] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243414:243414 [3] NCCL INFO Channel 10/0 : 3[3] -> 4[4] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243416:243416 [5] NCCL INFO Channel 04/0 : 5[5] -> 6[6] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243413:243413 [2] NCCL INFO Channel 08/0 : 2[2] -> 3[3] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243418:243418 [7] NCCL INFO Channel 05/0 : 7[7] -> 0[0] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243412:243412 [1] NCCL INFO Channel 07/0 : 1[1] -> 2[2] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243415:243415 [4] NCCL INFO Channel 05/0 : 4[4] -> 5[5] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243411:243411 [0] NCCL INFO Channel 05/0 : 0[0] -> 1[1] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243416:243416 [5] NCCL INFO Channel 05/0 : 5[5] -> 6[6] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243414:243414 [3] NCCL INFO Channel 11/0 : 3[3] -> 4[4] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243414:243414 [3] NCCL INFO Channel 12/0 : 3[3] -> 4[4] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243411:243411 [0] NCCL INFO Channel 06/0 : 0[0] -> 1[1] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243418:243418 [7] NCCL INFO Channel 06/0 : 7[7] -> 0[0] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243413:243413 [2] NCCL INFO Channel 09/0 : 2[2] -> 3[3] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243417:243417 [6] NCCL INFO Channel 07/0 : 6[6] -> 7[7] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243415:243415 [4] NCCL INFO Channel 06/0 : 4[4] -> 5[5] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243416:243416 [5] NCCL INFO Channel 06/0 : 5[5] -> 6[6] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243416:243416 [5] NCCL INFO Channel 07/0 : 5[5] -> 6[6] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243412:243412 [1] NCCL INFO Channel 08/0 : 1[1] -> 2[2] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243417:243417 [6] NCCL INFO Channel 08/0 : 6[6] -> 7[7] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243413:243413 [2] NCCL INFO Channel 10/0 : 2[2] -> 3[3] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243415:243415 [4] NCCL INFO Channel 07/0 : 4[4] -> 5[5] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243414:243414 [3] NCCL INFO Channel 13/0 : 3[3] -> 4[4] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243411:243411 [0] NCCL INFO Channel 07/0 : 0[0] -> 1[1] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243414:243414 [3] NCCL INFO Channel 14/0 : 3[3] -> 4[4] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243417:243417 [6] NCCL INFO Channel 09/0 : 6[6] -> 7[7] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243413:243413 [2] NCCL INFO Channel 11/0 : 2[2] -> 3[3] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243415:243415 [4] NCCL INFO Channel 08/0 : 4[4] -> 5[5] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243418:243418 [7] NCCL INFO Channel 07/0 : 7[7] -> 0[0] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243418:243418 [7] NCCL INFO Channel 08/0 : 7[7] -> 0[0] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243412:243412 [1] NCCL INFO Channel 09/0 : 1[1] -> 2[2] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243416:243416 [5] NCCL INFO Channel 08/0 : 5[5] -> 6[6] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243415:243415 [4] NCCL INFO Channel 09/0 : 4[4] -> 5[5] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243413:243413 [2] NCCL INFO Channel 12/0 : 2[2] -> 3[3] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243417:243417 [6] NCCL INFO Channel 10/0 : 6[6] -> 7[7] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243417:243417 [6] NCCL INFO Channel 11/0 : 6[6] -> 7[7] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243411:243411 [0] NCCL INFO Channel 08/0 : 0[0] -> 1[1] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243415:243415 [4] NCCL INFO Channel 10/0 : 4[4] -> 5[5] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243414:243414 [3] NCCL INFO Channel 15/0 : 3[3] -> 4[4] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243416:243416 [5] NCCL INFO Channel 09/0 : 5[5] -> 6[6] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243413:243413 [2] NCCL INFO Channel 13/0 : 2[2] -> 3[3] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243414:243414 [3] NCCL INFO Channel 16/0 : 3[3] -> 4[4] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243415:243415 [4] NCCL INFO Channel 11/0 : 4[4] -> 5[5] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243412:243412 [1] NCCL INFO Channel 10/0 : 1[1] -> 2[2] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243416:243416 [5] NCCL INFO Channel 10/0 : 5[5] -> 6[6] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243413:243413 [2] NCCL INFO Channel 14/0 : 2[2] -> 3[3] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243418:243418 [7] NCCL INFO Channel 09/0 : 7[7] -> 0[0] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243417:243417 [6] NCCL INFO Channel 12/0 : 6[6] -> 7[7] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243417:243417 [6] NCCL INFO Channel 13/0 : 6[6] -> 7[7] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243411:243411 [0] NCCL INFO Channel 09/0 : 0[0] -> 1[1] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243417:243417 [6] NCCL INFO Channel 14/0 : 6[6] -> 7[7] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243418:243418 [7] NCCL INFO Channel 10/0 : 7[7] -> 0[0] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243414:243414 [3] NCCL INFO Channel 17/0 : 3[3] -> 4[4] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243413:243413 [2] NCCL INFO Channel 15/0 : 2[2] -> 3[3] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243415:243415 [4] NCCL INFO Channel 12/0 : 4[4] -> 5[5] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243416:243416 [5] NCCL INFO Channel 11/0 : 5[5] -> 6[6] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243416:243416 [5] NCCL INFO Channel 12/0 : 5[5] -> 6[6] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243412:243412 [1] NCCL INFO Channel 11/0 : 1[1] -> 2[2] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243417:243417 [6] NCCL INFO Channel 15/0 : 6[6] -> 7[7] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243413:243413 [2] NCCL INFO Channel 16/0 : 2[2] -> 3[3] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243415:243415 [4] NCCL INFO Channel 13/0 : 4[4] -> 5[5] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243414:243414 [3] NCCL INFO Channel 18/0 : 3[3] -> 4[4] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243411:243411 [0] NCCL INFO Channel 10/0 : 0[0] -> 1[1] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243411:243411 [0] NCCL INFO Channel 11/0 : 0[0] -> 1[1] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243413:243413 [2] NCCL INFO Channel 17/0 : 2[2] -> 3[3] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243415:243415 [4] NCCL INFO Channel 14/0 : 4[4] -> 5[5] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243414:243414 [3] NCCL INFO Channel 19/0 : 3[3] -> 4[4] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243416:243416 [5] NCCL INFO Channel 13/0 : 5[5] -> 6[6] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243418:243418 [7] NCCL INFO Channel 11/0 : 7[7] -> 0[0] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243412:243412 [1] NCCL INFO Channel 12/0 : 1[1] -> 2[2] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243415:243415 [4] NCCL INFO Channel 15/0 : 4[4] -> 5[5] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243414:243414 [3] NCCL INFO Channel 20/0 : 3[3] -> 4[4] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243416:243416 [5] NCCL INFO Channel 14/0 : 5[5] -> 6[6] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243411:243411 [0] NCCL INFO Channel 12/0 : 0[0] -> 1[1] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243418:243418 [7] NCCL INFO Channel 12/0 : 7[7] -> 0[0] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243417:243417 [6] NCCL INFO Channel 16/0 : 6[6] -> 7[7] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243414:243414 [3] NCCL INFO Channel 21/0 : 3[3] -> 4[4] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243416:243416 [5] NCCL INFO Channel 15/0 : 5[5] -> 6[6] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243412:243412 [1] NCCL INFO Channel 13/0 : 1[1] -> 2[2] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243418:243418 [7] NCCL INFO Channel 13/0 : 7[7] -> 0[0] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243411:243411 [0] NCCL INFO Channel 13/0 : 0[0] -> 1[1] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243413:243413 [2] NCCL INFO Channel 18/0 : 2[2] -> 3[3] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243413:243413 [2] NCCL INFO Channel 19/0 : 2[2] -> 3[3] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243415:243415 [4] NCCL INFO Channel 16/0 : 4[4] -> 5[5] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243412:243412 [1] NCCL INFO Channel 14/0 : 1[1] -> 2[2] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243417:243417 [6] NCCL INFO Channel 17/0 : 6[6] -> 7[7] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243411:243411 [0] NCCL INFO Channel 14/0 : 0[0] -> 1[1] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243418:243418 [7] NCCL INFO Channel 14/0 : 7[7] -> 0[0] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243414:243414 [3] NCCL INFO Channel 22/0 : 3[3] -> 4[4] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243416:243416 [5] NCCL INFO Channel 16/0 : 5[5] -> 6[6] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243411:243411 [0] NCCL INFO Channel 15/0 : 0[0] -> 1[1] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243418:243418 [7] NCCL INFO Channel 15/0 : 7[7] -> 0[0] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243413:243413 [2] NCCL INFO Channel 20/0 : 2[2] -> 3[3] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243417:243417 [6] NCCL INFO Channel 18/0 : 6[6] -> 7[7] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243412:243412 [1] NCCL INFO Channel 15/0 : 1[1] -> 2[2] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243415:243415 [4] NCCL INFO Channel 17/0 : 4[4] -> 5[5] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243413:243413 [2] NCCL INFO Channel 21/0 : 2[2] -> 3[3] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243417:243417 [6] NCCL INFO Channel 19/0 : 6[6] -> 7[7] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243414:243414 [3] NCCL INFO Channel 23/0 : 3[3] -> 4[4] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243418:243418 [7] NCCL INFO Channel 16/0 : 7[7] -> 0[0] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243412:243412 [1] NCCL INFO Channel 16/0 : 1[1] -> 2[2] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243416:243416 [5] NCCL INFO Channel 17/0 : 5[5] -> 6[6] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243411:243411 [0] NCCL INFO Channel 16/0 : 0[0] -> 1[1] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243416:243416 [5] NCCL INFO Channel 18/0 : 5[5] -> 6[6] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243418:243418 [7] NCCL INFO Channel 17/0 : 7[7] -> 0[0] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243412:243412 [1] NCCL INFO Channel 17/0 : 1[1] -> 2[2] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243413:243413 [2] NCCL INFO Channel 22/0 : 2[2] -> 3[3] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243416:243416 [5] NCCL INFO Channel 19/0 : 5[5] -> 6[6] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243412:243412 [1] NCCL INFO Channel 18/0 : 1[1] -> 2[2] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243418:243418 [7] NCCL INFO Channel 18/0 : 7[7] -> 0[0] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243415:243415 [4] NCCL INFO Channel 18/0 : 4[4] -> 5[5] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243415:243415 [4] NCCL INFO Channel 19/0 : 4[4] -> 5[5] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243411:243411 [0] NCCL INFO Channel 17/0 : 0[0] -> 1[1] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243418:243418 [7] NCCL INFO Channel 19/0 : 7[7] -> 0[0] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243412:243412 [1] NCCL INFO Channel 19/0 : 1[1] -> 2[2] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243416:243416 [5] NCCL INFO Channel 20/0 : 5[5] -> 6[6] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243417:243417 [6] NCCL INFO Channel 20/0 : 6[6] -> 7[7] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243413:243413 [2] NCCL INFO Channel 23/0 : 2[2] -> 3[3] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243417:243417 [6] NCCL INFO Channel 21/0 : 6[6] -> 7[7] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243412:243412 [1] NCCL INFO Channel 20/0 : 1[1] -> 2[2] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243416:243416 [5] NCCL INFO Channel 21/0 : 5[5] -> 6[6] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243418:243418 [7] NCCL INFO Channel 20/0 : 7[7] -> 0[0] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243418:243418 [7] NCCL INFO Channel 21/0 : 7[7] -> 0[0] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243412:243412 [1] NCCL INFO Channel 21/0 : 1[1] -> 2[2] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243416:243416 [5] NCCL INFO Channel 22/0 : 5[5] -> 6[6] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243415:243415 [4] NCCL INFO Channel 20/0 : 4[4] -> 5[5] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243417:243417 [6] NCCL INFO Channel 22/0 : 6[6] -> 7[7] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243411:243411 [0] NCCL INFO Channel 18/0 : 0[0] -> 1[1] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243415:243415 [4] NCCL INFO Channel 21/0 : 4[4] -> 5[5] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243417:243417 [6] NCCL INFO Channel 23/0 : 6[6] -> 7[7] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243416:243416 [5] NCCL INFO Channel 23/0 : 5[5] -> 6[6] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243412:243412 [1] NCCL INFO Channel 22/0 : 1[1] -> 2[2] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243412:243412 [1] NCCL INFO Channel 23/0 : 1[1] -> 2[2] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243418:243418 [7] NCCL INFO Channel 22/0 : 7[7] -> 0[0] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243411:243411 [0] NCCL INFO Channel 19/0 : 0[0] -> 1[1] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243415:243415 [4] NCCL INFO Channel 22/0 : 4[4] -> 5[5] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243418:243418 [7] NCCL INFO Channel 23/0 : 7[7] -> 0[0] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243411:243411 [0] NCCL INFO Channel 20/0 : 0[0] -> 1[1] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243415:243415 [4] NCCL INFO Channel 23/0 : 4[4] -> 5[5] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243411:243411 [0] NCCL INFO Channel 21/0 : 0[0] -> 1[1] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243411:243411 [0] NCCL INFO Channel 22/0 : 0[0] -> 1[1] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243411:243411 [0] NCCL INFO Channel 23/0 : 0[0] -> 1[1] via P2P/CUMEM/read +ywang29-p4d-debug-1-worker-0:243416:243416 [5] NCCL INFO Connected all rings, use ring PXN 0 GDR 1 +ywang29-p4d-debug-1-worker-0:243417:243417 [6] NCCL INFO Connected all rings, use ring PXN 0 GDR 1 +ywang29-p4d-debug-1-worker-0:243415:243415 [4] NCCL INFO Connected all rings, use ring PXN 0 GDR 1 +ywang29-p4d-debug-1-worker-0:243412:243412 [1] NCCL INFO Connected all rings, use ring PXN 0 GDR 1 +ywang29-p4d-debug-1-worker-0:243418:243418 [7] NCCL INFO Connected all rings, use ring PXN 0 GDR 1 +ywang29-p4d-debug-1-worker-0:243414:243414 [3] NCCL INFO Connected all rings, use ring PXN 0 GDR 1 +ywang29-p4d-debug-1-worker-0:243413:243413 [2] NCCL INFO Connected all rings, use ring PXN 0 GDR 1 +ywang29-p4d-debug-1-worker-0:243411:243411 [0] NCCL INFO Connected all rings, use ring PXN 0 GDR 1 +Before initializing optimizer states +MA 4.73 GB Max_MA 4.73 GB CA 4.77 GB Max_CA 5 GB +CPU Virtual Memory: used = 57.11 GB, percent = 5.1% +After initializing optimizer states +MA 4.73 GB Max_MA 5.68 GB CA 5.71 GB Max_CA 6 GB +CPU Virtual Memory: used = 57.11 GB, percent = 5.1% +After initializing ZeRO optimizer +MA 4.73 GB Max_MA 4.73 GB CA 5.71 GB Max_CA 6 GB +CPU Virtual Memory: used = 57.11 GB, percent = 5.1% +[INFO|trainer.py:1587] 2026-04-29 17:06:18,875 >> ***** Running training ***** +[INFO|trainer.py:1588] 2026-04-29 17:06:18,875 >> Num examples = 30,000 +[INFO|trainer.py:1589] 2026-04-29 17:06:18,875 >> Num Epochs = 4 +[INFO|trainer.py:1590] 2026-04-29 17:06:18,875 >> Instantaneous batch size per device = 2 +[INFO|trainer.py:1593] 2026-04-29 17:06:18,875 >> Total train batch size (w. parallel, distributed & accumulation) = 128 +[INFO|trainer.py:1594] 2026-04-29 17:06:18,875 >> Gradient Accumulation steps = 8 +[INFO|trainer.py:1595] 2026-04-29 17:06:18,875 >> Total optimization steps = 940 +[INFO|trainer.py:1596] 2026-04-29 17:06:18,876 >> Number of trainable parameters = 2,031,739,904 +wandb: [wandb.login()] Loaded credentials for https://api.wandb.ai from WANDB_API_KEY. +wandb: Currently logged in as: kkhya (maskmoe) to https://api.wandb.ai. Use `wandb login --relogin` to force relogin +wandb: setting up run 152xmw6f +wandb: Tracking run with wandb version 0.26.1 +wandb: Run data is saved locally in /nfs/ywang29/lm-factory/wandb/run-20260429_170619-152xmw6f +wandb: Run `wandb offline` to turn off syncing. +wandb: Syncing run qwen3_1.7b_coding_fft +wandb: ⭐️ View project at https://wandb.ai/maskmoe/MFT-LM +wandb: 🚀 View run at https://wandb.ai/maskmoe/MFT-LM/runs/152xmw6f + 0%| | 0/940 [00:00> Saving model checkpoint to saves/qwen3_1.7b/coding/fft/checkpoint-500 +[INFO|configuration_utils.py:432] 2026-04-29 17:34:16,788 >> Configuration saved in saves/qwen3_1.7b/coding/fft/checkpoint-500/config.json +[INFO|configuration_utils.py:803] 2026-04-29 17:34:16,791 >> Configuration saved in saves/qwen3_1.7b/coding/fft/checkpoint-500/generation_config.json + + Writing model shards: 0%| | 0/1 [00:00> Model weights saved in saves/qwen3_1.7b/coding/fft/checkpoint-500/model.safetensors +[INFO|tokenization_utils_base.py:3224] 2026-04-29 17:34:23,025 >> chat template saved in saves/qwen3_1.7b/coding/fft/checkpoint-500/chat_template.jinja +[INFO|tokenization_utils_base.py:2078] 2026-04-29 17:34:23,028 >> tokenizer config file saved in saves/qwen3_1.7b/coding/fft/checkpoint-500/tokenizer_config.json + 53%|█████▎ | 501/940 [28:06<45:20, 6.20s/it] 53%|█████▎ | 502/940 [28:08<37:22, 5.12s/it] 54%|█████▎ | 503/940 [28:12<34:07, 4.69s/it] 54%|█████▎ | 504/940 [28:15<30:03, 4.14s/it] 54%|█████▎ | 505/940 [28:19<29:04, 4.01s/it] 54%|█████▍ | 506/940 [28:22<27:51, 3.85s/it] 54%|█████▍ | 507/940 [28:26<28:28, 3.95s/it] 54%|█████▍ | 508/940 [28:30<27:05, 3.76s/it] 54%|█████▍ | 509/940 [28:33<26:34, 3.70s/it] 54%|█████▍ | 510/940 [28:36<24:28, 3.41s/it] {'loss': '0.6106', 'grad_norm': '0.9619', 'learning_rate': '5.149e-06', 'epoch': '2.171'} + 54%|█████▍ | 510/940 [28:36<24:28, 3.41s/it] 54%|█████▍ | 511/940 [28:39<22:53, 3.20s/it] 54%|█████▍ | 512/940 [28:43<25:11, 3.53s/it] 55%|█████▍ | 513/940 [28:46<24:09, 3.40s/it] 55%|█████▍ | 514/940 [28:49<23:16, 3.28s/it] 55%|█████▍ | 515/940 [28:53<23:52, 3.37s/it] 55%|█████▍ | 516/940 [28:56<23:45, 3.36s/it] 55%|█████▌ | 517/940 [28:59<23:12, 3.29s/it] 55%|█████▌ | 518/940 [29:02<23:10, 3.30s/it] 55%|█████▌ | 519/940 [29:05<21:42, 3.09s/it] 55%|█████▌ | 520/940 [29:09<22:45, 3.25s/it] {'loss': '0.6105', 'grad_norm': '0.9641', 'learning_rate': '4.963e-06', 'epoch': '2.213'} + 55%|█████▌ | 520/940 [29:09<22:45, 3.25s/it] 55%|█████▌ | 521/940 [29:12<22:41, 3.25s/it] 56%|█████▌ | 522/940 [29:15<21:56, 3.15s/it] 56%|█████▌ | 523/940 [29:17<20:54, 3.01s/it] 56%|█████▌ | 524/940 [29:21<21:55, 3.16s/it] 56%|█████▌ | 525/940 [29:24<22:03, 3.19s/it] 56%|█████▌ | 526/940 [29:28<24:15, 3.52s/it] 56%|█████▌ | 527/940 [29:32<23:38, 3.44s/it] 56%|█████▌ | 528/940 [29:34<22:08, 3.22s/it] 56%|█████▋ | 529/940 [29:39<24:13, 3.54s/it] 56%|█████▋ | 530/940 [29:42<24:24, 3.57s/it] {'loss': '0.5798', 'grad_norm': '0.9159', 'learning_rate': '4.777e-06', 'epoch': '2.256'} + 56%|█████▋ | 530/940 [29:42<24:24, 3.57s/it] 56%|█████▋ | 531/940 [29:47<26:59, 3.96s/it] 57%|█████▋ | 532/940 [29:50<24:51, 3.66s/it] 57%|█████▋ | 533/940 [29:54<25:00, 3.69s/it] 57%|█████▋ | 534/940 [29:57<23:54, 3.53s/it] 57%|█████▋ | 535/940 [30:01<23:50, 3.53s/it] 57%|█████▋ | 536/940 [30:04<22:46, 3.38s/it] 57%|█████▋ | 537/940 [30:07<21:51, 3.25s/it] 57%|█████▋ | 538/940 [30:10<22:31, 3.36s/it] 57%|█████▋ | 539/940 [30:13<22:02, 3.30s/it] 57%|█████▋ | 540/940 [30:17<22:52, 3.43s/it] {'loss': '0.6197', 'grad_norm': '0.9406', 'learning_rate': '4.592e-06', 'epoch': '2.299'} + 57%|█████▋ | 540/940 [30:17<22:52, 3.43s/it] 58%|█████▊ | 541/940 [30:20<21:40, 3.26s/it] 58%|█████▊ | 542/940 [30:23<21:08, 3.19s/it] 58%|█████▊ | 543/940 [30:26<21:00, 3.18s/it] 58%|█████▊ | 544/940 [30:31<23:43, 3.59s/it] 58%|█████▊ | 545/940 [30:34<22:20, 3.39s/it] 58%|█████▊ | 546/940 [30:38<23:14, 3.54s/it] 58%|█████▊ | 547/940 [30:41<23:06, 3.53s/it] 58%|█████▊ | 548/940 [30:44<22:22, 3.42s/it] 58%|█████▊ | 549/940 [30:47<21:32, 3.31s/it] 59%|█████▊ | 550/940 [30:50<21:17, 3.28s/it] {'loss': '0.5832', 'grad_norm': '1.024', 'learning_rate': '4.407e-06', 'epoch': '2.341'} + 59%|█████▊ | 550/940 [30:50<21:17, 3.28s/it] 59%|█████▊ | 551/940 [30:53<20:25, 3.15s/it] 59%|█████▊ | 552/940 [30:56<19:29, 3.01s/it] 59%|█████▉ | 553/940 [30:59<19:46, 3.07s/it] 59%|█████▉ | 554/940 [31:03<21:38, 3.36s/it] 59%|█████▉ | 555/940 [31:06<21:03, 3.28s/it] 59%|█████▉ | 556/940 [31:09<20:28, 3.20s/it] 59%|█████▉ | 557/940 [31:12<19:50, 3.11s/it] 59%|█████▉ | 558/940 [31:17<23:44, 3.73s/it] 59%|█████▉ | 559/940 [31:21<23:26, 3.69s/it] 60%|█████▉ | 560/940 [31:24<22:30, 3.55s/it] {'loss': '0.5983', 'grad_norm': '0.9565', 'learning_rate': '4.223e-06', 'epoch': '2.384'} + 60%|█████▉ | 560/940 [31:24<22:30, 3.55s/it] 60%|█████▉ | 561/940 [31:28<23:22, 3.70s/it] 60%|█████▉ | 562/940 [31:32<22:59, 3.65s/it] 60%|█████▉ | 563/940 [31:36<23:16, 3.71s/it] 60%|██████ | 564/940 [31:39<21:53, 3.49s/it] 60%|██████ | 565/940 [31:42<20:53, 3.34s/it] 60%|██████ | 566/940 [31:45<21:24, 3.43s/it] 60%|██████ | 567/940 [31:50<23:30, 3.78s/it] 60%|██████ | 568/940 [31:53<21:26, 3.46s/it] 61%|██████ | 569/940 [31:55<20:09, 3.26s/it] 61%|██████ | 570/940 [32:00<21:57, 3.56s/it] {'loss': '0.5884', 'grad_norm': '0.9118', 'learning_rate': '4.04e-06', 'epoch': '2.427'} + 61%|██████ | 570/940 [32:00<21:57, 3.56s/it] 61%|██████ | 571/940 [32:03<21:48, 3.55s/it] 61%|██████ | 572/940 [32:07<22:46, 3.71s/it] 61%|██████ | 573/940 [32:11<23:02, 3.77s/it] 61%|██████ | 574/940 [32:14<20:56, 3.43s/it] 61%|██████ | 575/940 [32:18<22:58, 3.78s/it] 61%|██████▏ | 576/940 [32:22<23:20, 3.85s/it] 61%|██████▏ | 577/940 [32:25<21:37, 3.57s/it] 61%|██████▏ | 578/940 [32:28<20:15, 3.36s/it] 62%|██████▏ | 579/940 [32:31<19:02, 3.17s/it] 62%|██████▏ | 580/940 [32:34<19:33, 3.26s/it] {'loss': '0.5839', 'grad_norm': '0.9704', 'learning_rate': '3.859e-06', 'epoch': '2.469'} + 62%|██████▏ | 580/940 [32:34<19:33, 3.26s/it] 62%|██████▏ | 581/940 [32:37<19:06, 3.19s/it] 62%|██████▏ | 582/940 [32:41<19:42, 3.30s/it] 62%|██████▏ | 583/940 [32:44<19:04, 3.21s/it] 62%|██████▏ | 584/940 [32:47<17:49, 3.01s/it] 62%|██████▏ | 585/940 [32:49<17:31, 2.96s/it] 62%|██████▏ | 586/940 [32:53<18:49, 3.19s/it] 62%|██████▏ | 587/940 [32:57<20:17, 3.45s/it] 63%|██████▎ | 588/940 [33:01<20:14, 3.45s/it] 63%|██████▎ | 589/940 [33:04<20:29, 3.50s/it] 63%|██████▎ | 590/940 [33:07<18:45, 3.22s/it] {'loss': '0.6247', 'grad_norm': '0.9491', 'learning_rate': '3.679e-06', 'epoch': '2.512'} + 63%|██████▎ | 590/940 [33:07<18:45, 3.22s/it] 63%|██████▎ | 591/940 [33:12<22:45, 3.91s/it] 63%|██████▎ | 592/940 [33:16<21:35, 3.72s/it] 63%|██████▎ | 593/940 [33:20<22:16, 3.85s/it] 63%|██████▎ | 594/940 [33:26<25:41, 4.46s/it] 63%|██████▎ | 595/940 [33:29<23:34, 4.10s/it] 63%|██████▎ | 596/940 [33:32<22:36, 3.94s/it] 64%|██████▎ | 597/940 [33:36<22:19, 3.90s/it] 64%|██████▎ | 598/940 [33:41<22:57, 4.03s/it] 64%|██████▎ | 599/940 [33:45<23:02, 4.05s/it] 64%|██████▍ | 600/940 [33:48<21:18, 3.76s/it] {'loss': '0.6042', 'grad_norm': '0.9721', 'learning_rate': '3.501e-06', 'epoch': '2.555'} + 64%|██████▍ | 600/940 [33:48<21:18, 3.76s/it] 64%|██████▍ | 601/940 [33:50<19:29, 3.45s/it] 64%|██████▍ | 602/940 [33:55<20:54, 3.71s/it] 64%|██████▍ | 603/940 [33:58<20:40, 3.68s/it] 64%|██████▍ | 604/940 [34:01<18:44, 3.35s/it] 64%|██████▍ | 605/940 [34:04<18:38, 3.34s/it] 64%|██████▍ | 606/940 [34:09<20:07, 3.61s/it] 65%|██████▍ | 607/940 [34:11<18:16, 3.29s/it] 65%|██████▍ | 608/940 [34:14<18:02, 3.26s/it] 65%|██████▍ | 609/940 [34:18<18:09, 3.29s/it] 65%|██████▍ | 610/940 [34:21<18:50, 3.43s/it] {'loss': '0.6286', 'grad_norm': '1.117', 'learning_rate': '3.325e-06', 'epoch': '2.597'} + 65%|██████▍ | 610/940 [34:21<18:50, 3.43s/it] 65%|██████▌ | 611/940 [34:24<17:48, 3.25s/it] 65%|██████▌ | 612/940 [34:27<17:10, 3.14s/it] 65%|██████▌ | 613/940 [34:31<17:49, 3.27s/it] 65%|██████▌ | 614/940 [34:34<17:10, 3.16s/it] 65%|██████▌ | 615/940 [34:37<17:44, 3.28s/it] 66%|██████▌ | 616/940 [34:40<17:30, 3.24s/it] 66%|██████▌ | 617/940 [34:44<18:06, 3.36s/it] 66%|██████▌ | 618/940 [34:48<18:20, 3.42s/it] 66%|██████▌ | 619/940 [34:52<20:11, 3.77s/it] 66%|██████▌ | 620/940 [34:55<18:19, 3.44s/it] {'loss': '0.6007', 'grad_norm': '1.024', 'learning_rate': '3.151e-06', 'epoch': '2.64'} + 66%|██████▌ | 620/940 [34:55<18:19, 3.44s/it] 66%|██████▌ | 621/940 [34:58<17:11, 3.23s/it] 66%|██████▌ | 622/940 [35:00<15:39, 2.95s/it] 66%|██████▋ | 623/940 [35:03<15:47, 2.99s/it] 66%|██████▋ | 624/940 [35:07<17:35, 3.34s/it] 66%|██████▋ | 625/940 [35:11<18:00, 3.43s/it] 67%|██████▋ | 626/940 [35:14<17:10, 3.28s/it] 67%|██████▋ | 627/940 [35:17<17:12, 3.30s/it] 67%|██████▋ | 628/940 [35:20<16:09, 3.11s/it] 67%|██████▋ | 629/940 [35:22<15:13, 2.94s/it] 67%|██████▋ | 630/940 [35:25<14:30, 2.81s/it] {'loss': '0.5929', 'grad_norm': '1.057', 'learning_rate': '2.98e-06', 'epoch': '2.683'} + 67%|██████▋ | 630/940 [35:25<14:30, 2.81s/it] 67%|██████▋ | 631/940 [35:28<14:30, 2.82s/it] 67%|██████▋ | 632/940 [35:31<14:55, 2.91s/it] 67%|██████▋ | 633/940 [35:34<16:20, 3.19s/it] 67%|██████▋ | 634/940 [35:37<15:46, 3.09s/it] 68%|██████▊ | 635/940 [35:41<16:01, 3.15s/it] 68%|██████▊ | 636/940 [35:44<15:37, 3.08s/it] 68%|██████▊ | 637/940 [35:47<16:47, 3.32s/it] 68%|██████▊ | 638/940 [35:52<18:33, 3.69s/it] 68%|██████▊ | 639/940 [35:55<16:52, 3.36s/it] 68%|██████▊ | 640/940 [35:58<17:24, 3.48s/it] {'loss': '0.6104', 'grad_norm': '1.038', 'learning_rate': '2.811e-06', 'epoch': '2.725'} + 68%|██████▊ | 640/940 [35:58<17:24, 3.48s/it] 68%|██████▊ | 641/940 [36:01<15:41, 3.15s/it] 68%|██████▊ | 642/940 [36:04<15:55, 3.21s/it] 68%|██████▊ | 643/940 [36:08<17:19, 3.50s/it] 69%|██████▊ | 644/940 [36:11<16:19, 3.31s/it] 69%|██████▊ | 645/940 [36:14<15:47, 3.21s/it] 69%|██████▊ | 646/940 [36:17<15:16, 3.12s/it] 69%|██████▉ | 647/940 [36:21<16:14, 3.33s/it] 69%|██████▉ | 648/940 [36:24<16:04, 3.30s/it] 69%|██████▉ | 649/940 [36:28<16:49, 3.47s/it] 69%|██████▉ | 650/940 [36:31<16:48, 3.48s/it] {'loss': '0.5875', 'grad_norm': '0.9824', 'learning_rate': '2.646e-06', 'epoch': '2.768'} + 69%|██████▉ | 650/940 [36:31<16:48, 3.48s/it] 69%|██████▉ | 651/940 [36:34<15:08, 3.14s/it] 69%|██████▉ | 652/940 [36:37<14:42, 3.06s/it] 69%|██████▉ | 653/940 [36:40<15:18, 3.20s/it] 70%|██████▉ | 654/940 [36:43<15:01, 3.15s/it] 70%|██████▉ | 655/940 [36:46<14:03, 2.96s/it] 70%|██████▉ | 656/940 [36:49<14:31, 3.07s/it] 70%|██████▉ | 657/940 [36:51<13:29, 2.86s/it] 70%|███████ | 658/940 [36:55<13:48, 2.94s/it] 70%|███████ | 659/940 [36:59<15:39, 3.34s/it] 70%|███████ | 660/940 [37:04<18:00, 3.86s/it] {'loss': '0.5813', 'grad_norm': '1.017', 'learning_rate': '2.484e-06', 'epoch': '2.811'} + 70%|███████ | 660/940 [37:04<18:00, 3.86s/it] 70%|███████ | 661/940 [37:07<16:38, 3.58s/it] 70%|███████ | 662/940 [37:11<17:01, 3.67s/it] 71%|███████ | 663/940 [37:14<16:43, 3.62s/it] 71%|███████ | 664/940 [37:17<15:16, 3.32s/it] 71%|███████ | 665/940 [37:20<15:33, 3.39s/it] 71%|███████ | 666/940 [37:23<14:32, 3.19s/it] 71%|███████ | 667/940 [37:26<14:46, 3.25s/it] 71%|███████ | 668/940 [37:32<17:21, 3.83s/it] 71%|███████ | 669/940 [37:34<15:20, 3.40s/it] 71%|███████▏ | 670/940 [37:37<14:18, 3.18s/it] {'loss': '0.61', 'grad_norm': '1.189', 'learning_rate': '2.325e-06', 'epoch': '2.853'} + 71%|███████▏ | 670/940 [37:37<14:18, 3.18s/it] 71%|███████▏ | 671/940 [37:40<13:47, 3.08s/it] 71%|███████▏ | 672/940 [37:42<12:56, 2.90s/it] 72%|███████▏ | 673/940 [37:45<13:26, 3.02s/it] 72%|███████▏ | 674/940 [37:48<12:22, 2.79s/it] 72%|███████▏ | 675/940 [37:50<12:06, 2.74s/it] 72%|███████▏ | 676/940 [37:53<11:50, 2.69s/it] 72%|███████▏ | 677/940 [37:55<11:46, 2.69s/it] 72%|███████▏ | 678/940 [37:59<12:46, 2.93s/it] 72%|███████▏ | 679/940 [38:03<14:17, 3.28s/it] 72%|███████▏ | 680/940 [38:08<16:08, 3.72s/it] {'loss': '0.6121', 'grad_norm': '0.9742', 'learning_rate': '2.17e-06', 'epoch': '2.896'} + 72%|███████▏ | 680/940 [38:08<16:08, 3.72s/it] 72%|███████▏ | 681/940 [38:12<16:20, 3.79s/it] 73%|███████▎ | 682/940 [38:15<15:04, 3.51s/it] 73%|███████▎ | 683/940 [38:17<14:12, 3.32s/it] 73%|███████▎ | 684/940 [38:22<15:02, 3.52s/it] 73%|███████▎ | 685/940 [38:25<15:00, 3.53s/it] 73%|███████▎ | 686/940 [38:27<13:21, 3.16s/it] 73%|███████▎ | 687/940 [38:32<14:35, 3.46s/it] 73%|███████▎ | 688/940 [38:35<14:27, 3.44s/it] 73%|███████▎ | 689/940 [38:38<14:07, 3.38s/it] 73%|███████▎ | 690/940 [38:41<13:59, 3.36s/it] {'loss': '0.6172', 'grad_norm': '0.9061', 'learning_rate': '2.019e-06', 'epoch': '2.939'} + 73%|███████▎ | 690/940 [38:41<13:59, 3.36s/it] 74%|███████▎ | 691/940 [38:45<13:52, 3.35s/it] 74%|███████▎ | 692/940 [38:48<14:06, 3.41s/it] 74%|███████▎ | 693/940 [38:51<12:50, 3.12s/it] 74%|███████▍ | 694/940 [38:53<12:08, 2.96s/it] 74%|███████▍ | 695/940 [38:57<12:20, 3.02s/it] 74%|███████▍ | 696/940 [39:02<14:41, 3.61s/it] 74%|███████▍ | 697/940 [39:05<14:02, 3.47s/it] 74%|███████▍ | 698/940 [39:09<15:22, 3.81s/it] 74%|███████▍ | 699/940 [39:12<13:52, 3.45s/it] 74%|███████▍ | 700/940 [39:14<12:30, 3.13s/it] {'loss': '0.588', 'grad_norm': '1.029', 'learning_rate': '1.872e-06', 'epoch': '2.981'} + 74%|███████▍ | 700/940 [39:14<12:30, 3.13s/it] 75%|███████▍ | 701/940 [39:17<12:34, 3.16s/it] 75%|███████▍ | 702/940 [39:20<11:43, 2.95s/it] 75%|███████▍ | 703/940 [39:23<12:18, 3.12s/it] 75%|███████▍ | 704/940 [39:26<11:59, 3.05s/it] 75%|███████▌ | 705/940 [39:27<09:38, 2.46s/it] 75%|███████▌ | 706/940 [39:32<12:00, 3.08s/it] 75%|███████▌ | 707/940 [39:35<12:14, 3.15s/it] 75%|███████▌ | 708/940 [39:39<12:43, 3.29s/it] 75%|███████▌ | 709/940 [39:43<14:08, 3.67s/it] 76%|███████▌ | 710/940 [39:46<13:10, 3.44s/it] {'loss': '0.5795', 'grad_norm': '1.006', 'learning_rate': '1.73e-06', 'epoch': '3.021'} + 76%|███████▌ | 710/940 [39:46<13:10, 3.44s/it] 76%|███████▌ | 711/940 [39:49<12:12, 3.20s/it] 76%|███████▌ | 712/940 [39:53<12:56, 3.41s/it] 76%|███████▌ | 713/940 [39:56<12:16, 3.24s/it] 76%|███████▌ | 714/940 [39:58<11:34, 3.07s/it] 76%|███████▌ | 715/940 [40:01<11:00, 2.94s/it] 76%|███████▌ | 716/940 [40:05<12:03, 3.23s/it] 76%|███████▋ | 717/940 [40:09<12:26, 3.35s/it] 76%|███████▋ | 718/940 [40:12<12:37, 3.41s/it] 76%|███████▋ | 719/940 [40:15<12:11, 3.31s/it] 77%|███████▋ | 720/940 [40:18<11:50, 3.23s/it] {'loss': '0.5703', 'grad_norm': '0.975', 'learning_rate': '1.591e-06', 'epoch': '3.064'} + 77%|███████▋ | 720/940 [40:18<11:50, 3.23s/it] 77%|███████▋ | 721/940 [40:21<11:10, 3.06s/it] 77%|███████▋ | 722/940 [40:24<10:44, 2.96s/it] 77%|███████▋ | 723/940 [40:28<12:08, 3.36s/it] 77%|███████▋ | 724/940 [40:31<12:05, 3.36s/it] 77%|███████▋ | 725/940 [40:35<12:48, 3.57s/it] 77%|███████▋ | 726/940 [40:38<11:37, 3.26s/it] 77%|███████▋ | 727/940 [40:42<12:25, 3.50s/it] 77%|███████▋ | 728/940 [40:45<11:24, 3.23s/it] 78%|███████▊ | 729/940 [40:47<10:49, 3.08s/it] 78%|███████▊ | 730/940 [40:51<11:44, 3.35s/it] {'loss': '0.584', 'grad_norm': '0.9798', 'learning_rate': '1.458e-06', 'epoch': '3.107'} + 78%|███████▊ | 730/940 [40:51<11:44, 3.35s/it] 78%|███████▊ | 731/940 [40:54<11:08, 3.20s/it] 78%|███████▊ | 732/940 [40:57<10:17, 2.97s/it] 78%|███████▊ | 733/940 [41:00<10:43, 3.11s/it] 78%|███████▊ | 734/940 [41:04<11:40, 3.40s/it] 78%|███████▊ | 735/940 [41:08<11:49, 3.46s/it] 78%|███████▊ | 736/940 [41:10<11:00, 3.24s/it] 78%|███████▊ | 737/940 [41:14<11:18, 3.34s/it] 79%|███████▊ | 738/940 [41:17<11:26, 3.40s/it] 79%|███████▊ | 739/940 [41:21<11:23, 3.40s/it] 79%|███████▊ | 740/940 [41:24<11:28, 3.44s/it] {'loss': '0.5282', 'grad_norm': '1.004', 'learning_rate': '1.329e-06', 'epoch': '3.149'} + 79%|███████▊ | 740/940 [41:24<11:28, 3.44s/it] 79%|███████▉ | 741/940 [41:28<11:14, 3.39s/it] 79%|███████▉ | 742/940 [41:31<11:07, 3.37s/it] 79%|███████▉ | 743/940 [41:35<11:32, 3.52s/it] 79%|███████▉ | 744/940 [41:39<12:04, 3.70s/it] 79%|███████▉ | 745/940 [41:43<12:13, 3.76s/it] 79%|███████▉ | 746/940 [41:45<10:57, 3.39s/it] 79%|███████▉ | 747/940 [41:49<10:49, 3.37s/it] 80%|███████▉ | 748/940 [41:51<10:08, 3.17s/it] 80%|███████▉ | 749/940 [41:55<10:39, 3.35s/it] 80%|███████▉ | 750/940 [41:59<10:35, 3.34s/it] {'loss': '0.5751', 'grad_norm': '0.9754', 'learning_rate': '1.206e-06', 'epoch': '3.192'} + 80%|███████▉ | 750/940 [41:59<10:35, 3.34s/it] 80%|███████▉ | 751/940 [42:01<09:37, 3.06s/it] 80%|████████ | 752/940 [42:04<09:53, 3.16s/it] 80%|████████ | 753/940 [42:08<10:39, 3.42s/it] 80%|████████ | 754/940 [42:11<10:11, 3.29s/it] 80%|████████ | 755/940 [42:14<09:36, 3.12s/it] 80%|████████ | 756/940 [42:17<09:49, 3.20s/it] 81%|████████ | 757/940 [42:22<11:09, 3.66s/it] 81%|████████ | 758/940 [42:25<10:19, 3.41s/it] 81%|████████ | 759/940 [42:28<09:37, 3.19s/it] 81%|████████ | 760/940 [42:31<09:25, 3.14s/it] {'loss': '0.5495', 'grad_norm': '0.9909', 'learning_rate': '1.088e-06', 'epoch': '3.235'} + 81%|████████ | 760/940 [42:31<09:25, 3.14s/it] 81%|████████ | 761/940 [42:34<09:29, 3.18s/it] 81%|████████ | 762/940 [42:38<09:59, 3.37s/it] 81%|████████ | 763/940 [42:41<09:40, 3.28s/it] 81%|████████▏ | 764/940 [42:44<09:07, 3.11s/it] 81%|████████▏ | 765/940 [42:46<08:46, 3.01s/it] 81%|████████▏ | 766/940 [42:50<09:38, 3.32s/it] 82%|████████▏ | 767/940 [42:56<11:12, 3.89s/it] 82%|████████▏ | 768/940 [42:59<10:47, 3.77s/it] 82%|████████▏ | 769/940 [43:02<10:09, 3.56s/it] 82%|████████▏ | 770/940 [43:06<10:08, 3.58s/it] {'loss': '0.5537', 'grad_norm': '0.9958', 'learning_rate': '9.746e-07', 'epoch': '3.277'} + 82%|████████▏ | 770/940 [43:06<10:08, 3.58s/it] 82%|████████▏ | 771/940 [43:09<09:24, 3.34s/it] 82%|████████▏ | 772/940 [43:12<09:21, 3.34s/it] 82%|████████▏ | 773/940 [43:15<09:17, 3.34s/it] 82%|████████▏ | 774/940 [43:19<09:18, 3.36s/it] 82%|████████▏ | 775/940 [43:22<08:56, 3.25s/it] 83%|████████▎ | 776/940 [43:25<08:39, 3.17s/it] 83%|████████▎ | 777/940 [43:28<08:53, 3.27s/it] 83%|████████▎ | 778/940 [43:33<09:48, 3.63s/it] 83%|████████▎ | 779/940 [43:37<10:05, 3.76s/it] 83%|████████▎ | 780/940 [43:40<09:22, 3.51s/it] {'loss': '0.5587', 'grad_norm': '1.047', 'learning_rate': '8.673e-07', 'epoch': '3.32'} + 83%|████████▎ | 780/940 [43:40<09:22, 3.51s/it] 83%|████████▎ | 781/940 [43:42<08:33, 3.23s/it] 83%|████████▎ | 782/940 [43:45<08:23, 3.19s/it] 83%|████████▎ | 783/940 [43:48<08:00, 3.06s/it] 83%|████████▎ | 784/940 [43:50<07:25, 2.86s/it] 84%|████████▎ | 785/940 [43:53<07:16, 2.81s/it] 84%|████████▎ | 786/940 [43:56<07:11, 2.80s/it] 84%|████████▎ | 787/940 [44:02<09:37, 3.78s/it] 84%|████████▍ | 788/940 [44:04<08:35, 3.39s/it] 84%|████████▍ | 789/940 [44:08<08:36, 3.42s/it] 84%|████████▍ | 790/940 [44:11<08:17, 3.32s/it] {'loss': '0.5653', 'grad_norm': '0.9495', 'learning_rate': '7.657e-07', 'epoch': '3.363'} + 84%|████████▍ | 790/940 [44:11<08:17, 3.32s/it] 84%|████████▍ | 791/940 [44:15<08:25, 3.39s/it] 84%|████████▍ | 792/940 [44:19<08:49, 3.58s/it] 84%|████████▍ | 793/940 [44:21<08:02, 3.28s/it] 84%|████████▍ | 794/940 [44:24<07:38, 3.14s/it] 85%|████████▍ | 795/940 [44:29<08:50, 3.66s/it] 85%|████████▍ | 796/940 [44:33<09:23, 3.91s/it] 85%|████████▍ | 797/940 [44:37<09:12, 3.86s/it] 85%|████████▍ | 798/940 [44:40<08:10, 3.46s/it] 85%|████████▌ | 799/940 [44:44<08:41, 3.70s/it] 85%|████████▌ | 800/940 [44:48<08:48, 3.77s/it] {'loss': '0.5531', 'grad_norm': '1.351', 'learning_rate': '6.699e-07', 'epoch': '3.405'} + 85%|████████▌ | 800/940 [44:48<08:48, 3.77s/it] 85%|████████▌ | 801/940 [44:50<07:52, 3.40s/it] 85%|████████▌ | 802/940 [44:53<07:33, 3.29s/it] 85%|████████▌ | 803/940 [44:59<08:58, 3.93s/it] 86%|████████▌ | 804/940 [45:03<08:56, 3.94s/it] 86%|████████▌ | 805/940 [45:06<08:33, 3.80s/it] 86%|████████▌ | 806/940 [45:09<08:02, 3.60s/it] 86%|████████▌ | 807/940 [45:12<07:31, 3.39s/it] 86%|████████▌ | 808/940 [45:15<07:03, 3.21s/it] 86%|████████▌ | 809/940 [45:19<07:17, 3.34s/it] 86%|████████▌ | 810/940 [45:24<08:26, 3.90s/it] {'loss': '0.5778', 'grad_norm': '1.008', 'learning_rate': '5.8e-07', 'epoch': '3.448'} + 86%|████████▌ | 810/940 [45:24<08:26, 3.90s/it] 86%|████████▋ | 811/940 [45:31<10:07, 4.71s/it] 86%|████████▋ | 812/940 [45:35<09:49, 4.61s/it] 86%|████████▋ | 813/940 [45:38<08:57, 4.23s/it] 87%|████████▋ | 814/940 [45:42<08:44, 4.16s/it] 87%|████████▋ | 815/940 [45:45<07:57, 3.82s/it] 87%|████████▋ | 816/940 [45:49<07:43, 3.74s/it] 87%|████████▋ | 817/940 [45:51<06:44, 3.29s/it] 87%|████████▋ | 818/940 [45:55<07:05, 3.48s/it] 87%|████████▋ | 819/940 [45:57<06:20, 3.15s/it] 87%|████████▋ | 820/940 [46:01<06:34, 3.29s/it] {'loss': '0.5518', 'grad_norm': '0.9772', 'learning_rate': '4.963e-07', 'epoch': '3.491'} + 87%|████████▋ | 820/940 [46:01<06:34, 3.29s/it] 87%|████████▋ | 821/940 [46:04<06:27, 3.26s/it] 87%|████████▋ | 822/940 [46:07<06:20, 3.22s/it] 88%|████████▊ | 823/940 [46:10<05:44, 2.94s/it] 88%|████████▊ | 824/940 [46:15<06:57, 3.60s/it] 88%|████████▊ | 825/940 [46:17<06:20, 3.31s/it] 88%|████████▊ | 826/940 [46:20<05:59, 3.15s/it] 88%|████████▊ | 827/940 [46:23<05:35, 2.97s/it] 88%|████████▊ | 828/940 [46:26<05:37, 3.02s/it] 88%|████████▊ | 829/940 [46:28<05:17, 2.86s/it] 88%|████████▊ | 830/940 [46:31<04:59, 2.72s/it] {'loss': '0.5774', 'grad_norm': '1.047', 'learning_rate': '4.188e-07', 'epoch': '3.533'} + 88%|████████▊ | 830/940 [46:31<04:59, 2.72s/it] 88%|████████▊ | 831/940 [46:35<05:43, 3.15s/it] 89%|████████▊ | 832/940 [46:38<05:41, 3.16s/it] 89%|████████▊ | 833/940 [46:41<05:27, 3.07s/it] 89%|████████▊ | 834/940 [46:45<06:01, 3.41s/it] 89%|████████▉ | 835/940 [46:48<05:30, 3.15s/it] 89%|████████▉ | 836/940 [46:51<05:44, 3.32s/it] 89%|████████▉ | 837/940 [46:54<05:20, 3.11s/it] 89%|████████▉ | 838/940 [46:56<04:57, 2.92s/it] 89%|████████▉ | 839/940 [46:59<04:52, 2.89s/it] 89%|████████▉ | 840/940 [47:03<05:07, 3.07s/it] {'loss': '0.5664', 'grad_norm': '0.9108', 'learning_rate': '3.476e-07', 'epoch': '3.576'} + 89%|████████▉ | 840/940 [47:03<05:07, 3.07s/it] 89%|████████▉ | 841/940 [47:07<05:38, 3.42s/it] 90%|████████▉ | 842/940 [47:10<05:07, 3.13s/it] 90%|████████▉ | 843/940 [47:13<05:07, 3.17s/it] 90%|████████▉ | 844/940 [47:15<04:43, 2.95s/it] 90%|████████▉ | 845/940 [47:18<04:38, 2.93s/it] 90%|█████████ | 846/940 [47:21<04:42, 3.00s/it] 90%|█████████ | 847/940 [47:25<04:59, 3.22s/it] 90%|█████████ | 848/940 [47:29<05:06, 3.33s/it] 90%|█████████ | 849/940 [47:32<05:16, 3.48s/it] 90%|█████████ | 850/940 [47:36<05:14, 3.50s/it] {'loss': '0.5665', 'grad_norm': '0.987', 'learning_rate': '2.828e-07', 'epoch': '3.619'} + 90%|█████████ | 850/940 [47:36<05:14, 3.50s/it] 91%|█████████ | 851/940 [47:40<05:19, 3.59s/it] 91%|█████████ | 852/940 [47:42<04:51, 3.31s/it] 91%|█████████ | 853/940 [47:45<04:20, 2.99s/it] 91%|█████████ | 854/940 [47:47<04:06, 2.87s/it] 91%|█████████ | 855/940 [47:50<04:00, 2.83s/it] 91%|█████████ | 856/940 [47:53<03:52, 2.77s/it] 91%|█████████ | 857/940 [47:55<03:41, 2.67s/it] 91%|█████████▏| 858/940 [47:57<03:29, 2.56s/it] 91%|█████████▏| 859/940 [48:00<03:30, 2.60s/it] 91%|█████████▏| 860/940 [48:03<03:33, 2.67s/it] {'loss': '0.5847', 'grad_norm': '1.056', 'learning_rate': '2.245e-07', 'epoch': '3.661'} + 91%|█████████▏| 860/940 [48:03<03:33, 2.67s/it] 92%|█████████▏| 861/940 [48:06<03:30, 2.67s/it] 92%|█████████▏| 862/940 [48:09<03:36, 2.78s/it] 92%|█████████▏| 863/940 [48:12<03:42, 2.89s/it] 92%|█████████▏| 864/940 [48:14<03:34, 2.82s/it] 92%|█████████▏| 865/940 [48:17<03:32, 2.84s/it] 92%|█████████▏| 866/940 [48:20<03:25, 2.78s/it] 92%|█████████▏| 867/940 [48:23<03:25, 2.81s/it] 92%|█████████▏| 868/940 [48:26<03:32, 2.95s/it] 92%|█████████▏| 869/940 [48:29<03:38, 3.08s/it] 93%|█████████▎| 870/940 [48:33<03:36, 3.10s/it] {'loss': '0.5574', 'grad_norm': '1.035', 'learning_rate': '1.728e-07', 'epoch': '3.704'} + 93%|█████████▎| 870/940 [48:33<03:36, 3.10s/it] 93%|█████████▎| 871/940 [48:36<03:30, 3.05s/it] 93%|█████████▎| 872/940 [48:39<03:36, 3.19s/it] 93%|█████████▎| 873/940 [48:43<03:59, 3.57s/it] 93%|█████████▎| 874/940 [48:46<03:40, 3.34s/it] 93%|█████████▎| 875/940 [48:50<03:39, 3.38s/it] 93%|█████████▎| 876/940 [48:53<03:35, 3.36s/it] 93%|█████████▎| 877/940 [48:56<03:18, 3.14s/it] 93%|█████████▎| 878/940 [48:59<03:18, 3.21s/it] 94%|█████████▎| 879/940 [49:03<03:28, 3.41s/it] 94%|█████████▎| 880/940 [49:06<03:21, 3.36s/it] {'loss': '0.5604', 'grad_norm': '1.031', 'learning_rate': '1.277e-07', 'epoch': '3.747'} + 94%|█████████▎| 880/940 [49:06<03:21, 3.36s/it] 94%|█████████▎| 881/940 [49:09<03:03, 3.11s/it] 94%|█████████▍| 882/940 [49:12<02:59, 3.10s/it] 94%|█████████▍| 883/940 [49:14<02:48, 2.95s/it] 94%|█████████▍| 884/940 [49:19<03:05, 3.31s/it] 94%|█████████▍| 885/940 [49:21<02:47, 3.04s/it] 94%|█████████▍| 886/940 [49:25<02:56, 3.28s/it] 94%|█████████▍| 887/940 [49:28<02:54, 3.29s/it] 94%|█████████▍| 888/940 [49:33<03:13, 3.73s/it] 95%|█████████▍| 889/940 [49:36<03:02, 3.59s/it] 95%|█████████▍| 890/940 [49:39<02:43, 3.27s/it] {'loss': '0.5812', 'grad_norm': '0.9566', 'learning_rate': '8.94e-08', 'epoch': '3.789'} + 95%|█████████▍| 890/940 [49:39<02:43, 3.27s/it] 95%|█████████▍| 891/940 [49:41<02:31, 3.09s/it] 95%|█████████▍| 892/940 [49:44<02:27, 3.08s/it] 95%|█████████▌| 893/940 [49:48<02:25, 3.09s/it] 95%|█████████▌| 894/940 [49:51<02:34, 3.36s/it] 95%|█████████▌| 895/940 [49:55<02:36, 3.49s/it] 95%|█████████▌| 896/940 [49:58<02:25, 3.30s/it] 95%|█████████▌| 897/940 [50:01<02:16, 3.17s/it] 96%|█████████▌| 898/940 [50:05<02:19, 3.32s/it] 96%|█████████▌| 899/940 [50:08<02:22, 3.46s/it] 96%|█████████▌| 900/940 [50:13<02:25, 3.64s/it] {'loss': '0.5552', 'grad_norm': '0.9202', 'learning_rate': '5.784e-08', 'epoch': '3.832'} + 96%|█████████▌| 900/940 [50:13<02:25, 3.64s/it] 96%|█████████▌| 901/940 [50:17<02:29, 3.84s/it] 96%|█████████▌| 902/940 [50:21<02:32, 4.02s/it] 96%|█████████▌| 903/940 [50:24<02:18, 3.74s/it] 96%|█████████▌| 904/940 [50:27<02:02, 3.40s/it] 96%|█████████▋| 905/940 [50:32<02:11, 3.74s/it] 96%|█████████▋| 906/940 [50:36<02:10, 3.84s/it] 96%|█████████▋| 907/940 [50:40<02:09, 3.93s/it] 97%|█████████▋| 908/940 [50:42<01:52, 3.51s/it] 97%|█████████▋| 909/940 [50:45<01:41, 3.29s/it] 97%|█████████▋| 910/940 [50:49<01:45, 3.53s/it] {'loss': '0.5619', 'grad_norm': '0.9611', 'learning_rate': '3.309e-08', 'epoch': '3.875'} + 97%|█████████▋| 910/940 [50:49<01:45, 3.53s/it] 97%|█████████▋| 911/940 [50:53<01:49, 3.77s/it] 97%|█████████▋| 912/940 [50:57<01:47, 3.85s/it] 97%|█████████▋| 913/940 [51:01<01:42, 3.79s/it] 97%|█████████▋| 914/940 [51:04<01:30, 3.48s/it] 97%|█████████▋| 915/940 [51:08<01:29, 3.58s/it] 97%|█████████▋| 916/940 [51:12<01:29, 3.72s/it] 98%|█████████▊| 917/940 [51:15<01:19, 3.46s/it] 98%|█████████▊| 918/940 [51:18<01:16, 3.49s/it] 98%|█████████▊| 919/940 [51:21<01:06, 3.19s/it] 98%|█████████▊| 920/940 [51:24<01:04, 3.21s/it] {'loss': '0.5391', 'grad_norm': '0.9684', 'learning_rate': '1.52e-08', 'epoch': '3.917'} + 98%|█████████▊| 920/940 [51:24<01:04, 3.21s/it] 98%|█████████▊| 921/940 [51:27<01:00, 3.18s/it] 98%|█████████▊| 922/940 [51:30<00:53, 2.99s/it] 98%|█████████▊| 923/940 [51:32<00:48, 2.86s/it] 98%|█████████▊| 924/940 [51:36<00:51, 3.19s/it] 98%|█████████▊| 925/940 [51:39<00:44, 3.00s/it] 99%|█████████▊| 926/940 [51:42<00:43, 3.08s/it] 99%|█████████▊| 927/940 [51:45<00:39, 3.03s/it] 99%|█████████▊| 928/940 [51:48<00:36, 3.08s/it] 99%|█████████▉| 929/940 [51:51<00:33, 3.07s/it] 99%|█████████▉| 930/940 [51:54<00:29, 2.94s/it] {'loss': '0.5671', 'grad_norm': '0.9603', 'learning_rate': '4.171e-09', 'epoch': '3.96'} + 99%|█████████▉| 930/940 [51:54<00:29, 2.94s/it] 99%|█████████▉| 931/940 [51:56<00:25, 2.83s/it] 99%|█████████▉| 932/940 [51:59<00:21, 2.70s/it] 99%|█████████▉| 933/940 [52:02<00:20, 2.98s/it] 99%|█████████▉| 934/940 [52:05<00:17, 2.83s/it] 99%|█████████▉| 935/940 [52:08<00:14, 2.82s/it] 100%|█████████▉| 936/940 [52:12<00:13, 3.30s/it] 100%|█████████▉| 937/940 [52:15<00:09, 3.13s/it] 100%|█████████▉| 938/940 [52:20<00:07, 3.88s/it] 100%|█████████▉| 939/940 [52:23<00:03, 3.50s/it] 100%|██████████| 940/940 [52:24<00:00, 2.76s/it] {'loss': '0.521', 'grad_norm': '1.651', 'learning_rate': '3.447e-11', 'epoch': '4'} + 100%|██████████| 940/940 [52:24<00:00, 2.76s/it][INFO|trainer.py:3797] 2026-04-29 17:58:48,007 >> Saving model checkpoint to saves/qwen3_1.7b/coding/fft/checkpoint-940 +[INFO|configuration_utils.py:432] 2026-04-29 17:58:48,014 >> Configuration saved in saves/qwen3_1.7b/coding/fft/checkpoint-940/config.json +[INFO|configuration_utils.py:803] 2026-04-29 17:58:48,018 >> Configuration saved in saves/qwen3_1.7b/coding/fft/checkpoint-940/generation_config.json + + Writing model shards: 0%| | 0/1 [00:00> Model weights saved in saves/qwen3_1.7b/coding/fft/checkpoint-940/model.safetensors +[INFO|tokenization_utils_base.py:3224] 2026-04-29 17:58:54,112 >> chat template saved in saves/qwen3_1.7b/coding/fft/checkpoint-940/chat_template.jinja +[INFO|tokenization_utils_base.py:2078] 2026-04-29 17:58:54,115 >> tokenizer config file saved in saves/qwen3_1.7b/coding/fft/checkpoint-940/tokenizer_config.json +[INFO|trainer.py:1863] 2026-04-29 17:58:54,274 >> + +Training completed. Do not forget to share your model on huggingface.co/models =) + + + {'train_runtime': '3155', 'train_samples_per_second': '38.03', 'train_steps_per_second': '0.298', 'train_loss': '0.6604', 'epoch': '4'} + 100%|██████████| 940/940 [52:33<00:00, 2.76s/it] 100%|██████████| 940/940 [52:33<00:00, 3.36s/it] +[INFO|trainer.py:3797] 2026-04-29 17:58:55,513 >> Saving model checkpoint to saves/qwen3_1.7b/coding/fft +[INFO|configuration_utils.py:432] 2026-04-29 17:58:55,520 >> Configuration saved in saves/qwen3_1.7b/coding/fft/config.json +[INFO|configuration_utils.py:803] 2026-04-29 17:58:55,525 >> Configuration saved in saves/qwen3_1.7b/coding/fft/generation_config.json + Writing model shards: 0%| | 0/1 [00:00> Model weights saved in saves/qwen3_1.7b/coding/fft/model.safetensors +[INFO|tokenization_utils_base.py:3224] 2026-04-29 17:59:01,701 >> chat template saved in saves/qwen3_1.7b/coding/fft/chat_template.jinja +[INFO|tokenization_utils_base.py:2078] 2026-04-29 17:59:01,704 >> tokenizer config file saved in saves/qwen3_1.7b/coding/fft/tokenizer_config.json +ywang29-p4d-debug-1-worker-0:243413:243413 [2] NCCL INFO comm 0x55b7e396d9c0 rank 2 nranks 8 cudaDev 2 busId 201c0 - Destroy COMPLETE +ywang29-p4d-debug-1-worker-0:243414:243414 [3] NCCL INFO comm 0x55ae95eb9e30 rank 3 nranks 8 cudaDev 3 busId 201d0 - Destroy COMPLETE +ywang29-p4d-debug-1-worker-0:243416:243416 [5] NCCL INFO comm 0x558fdd7d54e0 rank 5 nranks 8 cudaDev 5 busId 901d0 - Destroy COMPLETE +ywang29-p4d-debug-1-worker-0:243415:243415 [4] NCCL INFO comm 0x561d4c169060 rank 4 nranks 8 cudaDev 4 busId 901c0 - Destroy COMPLETE +ywang29-p4d-debug-1-worker-0:243417:243417 [6] NCCL INFO comm 0x55b500e80270 rank 6 nranks 8 cudaDev 6 busId a01c0 - Destroy COMPLETE +ywang29-p4d-debug-1-worker-0:243416:243416 [5] NCCL INFO comm 0x558fd1beff10 rank 5 nranks 8 cudaDev 5 busId 901d0 - Destroy COMPLETE +ywang29-p4d-debug-1-worker-0:243415:243415 [4] NCCL INFO comm 0x561d46f046a0 rank 4 nranks 8 cudaDev 4 busId 901c0 - Destroy COMPLETE +ywang29-p4d-debug-1-worker-0:243414:243414 [3] NCCL INFO comm 0x55ae88a9dd90 rank 3 nranks 8 cudaDev 3 busId 201d0 - Destroy COMPLETE +[INFO|trainer.py:3797] 2026-04-29 17:59:04,829 >> Saving model checkpoint to saves/qwen3_1.7b/coding/fft +[INFO|configuration_utils.py:432] 2026-04-29 17:59:04,837 >> Configuration saved in saves/qwen3_1.7b/coding/fft/config.json +[INFO|configuration_utils.py:803] 2026-04-29 17:59:04,840 >> Configuration saved in saves/qwen3_1.7b/coding/fft/generation_config.json + Writing model shards: 0%| | 0/1 [00:00> Model weights saved in saves/qwen3_1.7b/coding/fft/model.safetensors +[INFO|tokenization_utils_base.py:3224] 2026-04-29 17:59:11,820 >> chat template saved in saves/qwen3_1.7b/coding/fft/chat_template.jinja +[INFO|tokenization_utils_base.py:2078] 2026-04-29 17:59:11,823 >> tokenizer config file saved in saves/qwen3_1.7b/coding/fft/tokenizer_config.json +[INFO|modelcard.py:266] 2026-04-29 17:59:12,253 >> Dropping the following result as it does not have all the necessary fields: +{'task': {'name': 'Causal Language Modeling', 'type': 'text-generation'}} + Processing Files (0 / 0) : | | 0.00B / 0.00B + New Data Upload : | | 0.00B / 0.00B  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 0%| | 29.6kB / 4.06GB  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 0%| | 29.6kB / 4.06GB  Processing Files (1 / 2) : 0%| | 11.5MB / 4.07GB, 57.1MB/s + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 0%| | 29.6kB / 4.06GB  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 0%| | 29.6kB / 4.06GB  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 0%| | 29.6kB / 4.06GB  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 0%| | 636kB / 4.06GB  Processing Files (1 / 2) : 0%| | 12.1MB / 4.07GB, 12.1MB/s + New Data Upload : 0%| | 607kB / 134MB, 606kB/s  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 0%| | 1.24MB / 4.06GB  Processing Files (1 / 2) : 0%| | 12.7MB / 4.07GB, 10.6MB/s + New Data Upload : 1%| | 1.21MB / 134MB, 1.01MB/s  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 0%| | 8.52MB / 4.06GB  Processing Files (1 / 2) : 0%| | 19.9MB / 4.07GB, 14.2MB/s + New Data Upload : 6%|▋ | 8.49MB / 134MB, 6.06MB/s  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 1%| | 21.3MB / 4.06GB  Processing Files (1 / 2) : 1%| | 32.7MB / 4.07GB, 20.4MB/s + New Data Upload : 16%|█▌ | 21.2MB / 134MB, 13.3MB/s  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 1%| | 35.8MB / 4.06GB  Processing Files (1 / 2) : 1%| | 47.2MB / 4.07GB, 26.2MB/s + New Data Upload : 27%|██▋ | 35.8MB / 134MB, 19.9MB/s  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 1%| | 44.9MB / 4.06GB  Processing Files (1 / 2) : 1%|▏ | 56.3MB / 4.07GB, 28.2MB/s + New Data Upload : 33%|███▎ | 44.9MB / 134MB, 22.4MB/s  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 1%|▏ | 58.9MB / 4.06GB  Processing Files (1 / 2) : 2%|▏ | 70.3MB / 4.07GB, 31.9MB/s + New Data Upload : 44%|████▍ | 58.8MB / 134MB, 26.7MB/s  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 2%|▏ | 66.7MB / 4.06GB  Processing Files (1 / 2) : 2%|▏ | 78.2MB / 4.07GB, 32.6MB/s + New Data Upload : 50%|████▉ | 66.7MB / 134MB, 27.8MB/s  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 2%|▏ | 66.7MB / 4.06GB  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 2%|▏ | 66.7MB / 4.06GB  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 2%|▏ | 66.7MB / 4.06GB  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 2%|▏ | 66.7MB / 4.06GB  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 2%|▏ | 71.3MB / 4.06GB  Processing Files (1 / 2) : 2%|▏ | 82.7MB / 4.07GB, 24.3MB/s + New Data Upload : 35%|███▌ | 71.2MB / 201MB, 20.9MB/s  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 2%|▏ | 84.0MB / 4.06GB  Processing Files (1 / 2) : 2%|▏ | 95.4MB / 4.07GB, 26.5MB/s + New Data Upload : 42%|████▏ | 83.9MB / 201MB, 23.3MB/s  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 2%|▏ | 98.5MB / 4.06GB  Processing Files (1 / 2) : 3%|▎ | 110MB / 4.07GB, 28.9MB/s + New Data Upload : 37%|███▋ | 98.5MB / 268MB, 25.9MB/s  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 3%|▎ | 109MB / 4.06GB  Processing Files (1 / 2) : 3%|▎ | 120MB / 4.07GB, 30.1MB/s + New Data Upload : 41%|████ | 109MB / 268MB, 27.2MB/s  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 3%|▎ | 134MB / 4.06GB  Processing Files (1 / 2) : 4%|▎ | 145MB / 4.07GB, 34.5MB/s + New Data Upload : 50%|████▉ | 134MB / 268MB, 31.8MB/s  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 4%|▍ | 160MB / 4.06GB  Processing Files (1 / 2) : 4%|▍ | 172MB / 4.07GB, 39.0MB/s + New Data Upload : 48%|████▊ | 160MB / 335MB, 36.4MB/s  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 4%|▍ | 177MB / 4.06GB  Processing Files (1 / 2) : 5%|▍ | 188MB / 4.07GB, 40.9MB/s + New Data Upload : 44%|████▍ | 177MB / 402MB, 38.4MB/s  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 5%|▍ | 199MB / 4.06GB  Processing Files (1 / 2) : 5%|▌ | 211MB / 4.07GB, 43.9MB/s + New Data Upload : 49%|████▉ | 199MB / 402MB, 41.5MB/s  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 6%|▌ | 227MB / 4.06GB  Processing Files (1 / 2) : 6%|▌ | 238MB / 4.07GB, 47.7MB/s + New Data Upload : 48%|████▊ | 227MB / 469MB, 45.4MB/s  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 6%|▌ | 246MB / 4.06GB  Processing Files (1 / 2) : 6%|▋ | 258MB / 4.07GB, 49.6MB/s + New Data Upload : 52%|█████▏ | 246MB / 469MB, 47.4MB/s  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 7%|▋ | 268MB / 4.06GB  Processing Files (1 / 2) : 7%|▋ | 280MB / 4.07GB, 51.8MB/s + New Data Upload : 57%|█████▋ | 268MB / 469MB, 49.7MB/s  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 7%|▋ | 294MB / 4.06GB  Processing Files (1 / 2) : 7%|▋ | 305MB / 4.07GB, 54.5MB/s + New Data Upload : 63%|██████▎ | 294MB / 469MB, 52.4MB/s  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 8%|▊ | 321MB / 4.06GB  Processing Files (1 / 2) : 8%|▊ | 332MB / 4.07GB, 57.3MB/s + New Data Upload : 60%|█████▉ | 321MB / 536MB, 55.3MB/s  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 9%|▊ | 349MB / 4.06GB  Processing Files (1 / 2) : 9%|▉ | 360MB / 4.07GB, 60.1MB/s + New Data Upload : 65%|██████▌ | 349MB / 536MB, 58.2MB/s  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 9%|▉ | 381MB / 4.06GB  Processing Files (1 / 2) : 10%|▉ | 392MB / 4.07GB, 63.2MB/s + New Data Upload : 63%|██████▎ | 380MB / 603MB, 61.4MB/s  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 10%|█ | 416MB / 4.06GB  Processing Files (1 / 2) : 10%|█ | 428MB / 4.07GB, 66.8MB/s + New Data Upload : 69%|██████▉ | 416MB / 603MB, 65.1MB/s  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 11%|█ | 447MB / 4.06GB  Processing Files (1 / 2) : 11%|█ | 458MB / 4.07GB, 69.4MB/s + New Data Upload : 74%|███████▍ | 447MB / 603MB, 67.7MB/s  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 12%|█▏ | 476MB / 4.06GB  Processing Files (1 / 2) : 12%|█▏ | 488MB / 4.07GB, 71.7MB/s + New Data Upload : 71%|███████ | 476MB / 671MB, 70.0MB/s  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 13%|█▎ | 510MB / 4.06GB  Processing Files (1 / 2) : 13%|█▎ | 521MB / 4.07GB, 74.4MB/s + New Data Upload : 76%|███████▌ | 509MB / 671MB, 72.8MB/s  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 13%|█▎ | 542MB / 4.06GB  Processing Files (1 / 2) : 14%|█▎ | 554MB / 4.07GB, 76.9MB/s + New Data Upload : 81%|████████ | 542MB / 671MB, 75.3MB/s  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 14%|█▍ | 573MB / 4.06GB  Processing Files (1 / 2) : 14%|█▍ | 584MB / 4.07GB, 79.0MB/s + New Data Upload : 85%|████████▌ | 573MB / 671MB, 77.4MB/s  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 15%|█▍ | 590MB / 4.06GB  Processing Files (1 / 2) : 15%|█▍ | 602MB / 4.07GB, 79.2MB/s + New Data Upload : 80%|███████▉ | 590MB / 738MB, 77.6MB/s  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 15%|█▌ | 614MB / 4.06GB  Processing Files (1 / 2) : 15%|█▌ | 626MB / 4.07GB, 80.2MB/s + New Data Upload : 76%|███████▋ | 614MB / 805MB, 78.7MB/s  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 16%|█▌ | 637MB / 4.06GB  Processing Files (1 / 2) : 16%|█▌ | 649MB / 4.07GB, 81.1MB/s + New Data Upload : 79%|███████▉ | 637MB / 805MB, 79.7MB/s  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 17%|█▋ | 676MB / 4.06GB  Processing Files (1 / 2) : 17%|█▋ | 688MB / 4.07GB, 83.8MB/s + New Data Upload : 84%|████████▍ | 676MB / 805MB, 82.4MB/s  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 17%|█▋ | 705MB / 4.06GB  Processing Files (1 / 2) : 18%|█▊ | 717MB / 4.07GB, 85.3MB/s + New Data Upload : 81%|████████ | 705MB / 872MB, 84.0MB/s  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 18%|█▊ | 738MB / 4.06GB  Processing Files (1 / 2) : 18%|█▊ | 749MB / 4.07GB, 87.1MB/s + New Data Upload : 79%|███████▊ | 737MB / 939MB, 85.8MB/s  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 19%|█▉ | 770MB / 4.06GB  Processing Files (1 / 2) : 19%|█▉ | 781MB / 4.07GB, 88.8MB/s + New Data Upload : 82%|████████▏ | 770MB / 939MB, 87.4MB/s  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 20%|█▉ | 805MB / 4.06GB  Processing Files (1 / 2) : 20%|██ | 816MB / 4.07GB, 90.6MB/s + New Data Upload : 86%|████████▌ | 804MB / 939MB, 89.3MB/s  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 20%|██ | 826MB / 4.06GB  Processing Files (1 / 2) : 21%|██ | 837MB / 4.07GB, 91.0MB/s + New Data Upload : 82%|████████▏ | 825MB / 1.01GB, 89.7MB/s  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 22%|██▏ | 902MB / 4.06GB  Processing Files (1 / 2) : 22%|██▏ | 914MB / 4.07GB, 97.2MB/s + New Data Upload : 86%|████████▌ | 860MB / 1.01GB, 91.5MB/s  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 24%|██▍ | 974MB / 4.06GB  Processing Files (1 / 2) : 24%|██▍ | 986MB / 4.07GB, 103MB/s + New Data Upload : 84%|████████▍ | 901MB / 1.07GB, 93.9MB/s  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 26%|██▌ | 1.06GB / 4.06GB  Processing Files (1 / 2) : 26%|██▋ | 1.07GB / 4.07GB, 109MB/s + New Data Upload : 87%|████████▋ | 933MB / 1.07GB, 95.2MB/s  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 27%|██▋ | 1.11GB / 4.06GB  Processing Files (1 / 2) : 27%|██▋ | 1.12GB / 4.07GB, 112MB/s + New Data Upload : 85%|████████▍ | 967MB / 1.14GB, 96.7MB/s  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 28%|██▊ | 1.15GB / 4.06GB  Processing Files (1 / 2) : 28%|██▊ | 1.16GB / 4.07GB, 114MB/s + New Data Upload : 88%|████████▊ | 1.01GB / 1.14GB, 98.5MB/s  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 29%|██▉ | 1.18GB / 4.06GB  Processing Files (1 / 2) : 29%|██▉ | 1.19GB / 4.07GB, 116MB/s + New Data Upload : 86%|████████▌ | 1.04GB / 1.21GB, 102MB/s  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 30%|██▉ | 1.20GB / 4.06GB  Processing Files (1 / 2) : 30%|██▉ | 1.21GB / 4.07GB, 118MB/s + New Data Upload : 83%|████████▎ | 1.06GB / 1.27GB, 104MB/s  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 30%|███ | 1.23GB / 4.06GB  Processing Files (1 / 2) : 30%|███ | 1.24GB / 4.07GB, 120MB/s + New Data Upload : 85%|████████▌ | 1.08GB / 1.27GB, 106MB/s  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 31%|███ | 1.25GB / 4.06GB  Processing Files (1 / 2) : 31%|███ | 1.27GB / 4.07GB, 123MB/s + New Data Upload : 83%|████████▎ | 1.11GB / 1.34GB, 109MB/s  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 31%|███▏ | 1.27GB / 4.06GB  Processing Files (1 / 2) : 31%|███▏ | 1.28GB / 4.07GB, 125MB/s + New Data Upload : 84%|████████▍ | 1.13GB / 1.34GB, 111MB/s  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 32%|███▏ | 1.30GB / 4.06GB  Processing Files (1 / 2) : 32%|███▏ | 1.32GB / 4.07GB, 128MB/s + New Data Upload : 83%|████████▎ | 1.16GB / 1.41GB, 114MB/s  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 33%|███▎ | 1.35GB / 4.06GB  Processing Files (1 / 2) : 33%|███▎ | 1.36GB / 4.07GB, 131MB/s + New Data Upload : 86%|████████▌ | 1.21GB / 1.41GB, 118MB/s  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 34%|███▍ | 1.39GB / 4.06GB  Processing Files (1 / 2) : 34%|███▍ | 1.40GB / 4.07GB, 134MB/s + New Data Upload : 85%|████████▍ | 1.25GB / 1.48GB, 121MB/s  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 35%|███▌ | 1.44GB / 4.06GB  Processing Files (1 / 2) : 36%|███▌ | 1.45GB / 4.07GB, 138MB/s + New Data Upload : 88%|████████▊ | 1.30GB / 1.48GB, 124MB/s  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 36%|███▋ | 1.47GB / 4.06GB  Processing Files (1 / 2) : 36%|███▋ | 1.48GB / 4.07GB, 140MB/s + New Data Upload : 86%|████████▋ | 1.33GB / 1.54GB, 126MB/s  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 37%|███▋ | 1.50GB / 4.06GB  Processing Files (1 / 2) : 37%|███▋ | 1.51GB / 4.07GB, 141MB/s + New Data Upload : 88%|████████▊ | 1.36GB / 1.54GB, 128MB/s  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 38%|███▊ | 1.54GB / 4.06GB  Processing Files (1 / 2) : 38%|███▊ | 1.55GB / 4.07GB, 144MB/s + New Data Upload : 87%|████████▋ | 1.39GB / 1.61GB, 130MB/s  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 39%|███▉ | 1.58GB / 4.06GB  Processing Files (1 / 2) : 39%|███▉ | 1.59GB / 4.07GB, 148MB/s + New Data Upload : 89%|████████▉ | 1.43GB / 1.61GB, 134MB/s  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 40%|███▉ | 1.61GB / 4.06GB  Processing Files (1 / 2) : 40%|███▉ | 1.62GB / 4.07GB, 151MB/s + New Data Upload : 88%|████████▊ | 1.47GB / 1.68GB, 137MB/s  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 41%|████ | 1.66GB / 4.06GB  Processing Files (1 / 2) : 41%|████ | 1.67GB / 4.07GB, 156MB/s + New Data Upload : 90%|█████████ | 1.52GB / 1.68GB, 142MB/s  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 42%|████▏ | 1.69GB / 4.06GB  Processing Files (1 / 2) : 42%|████▏ | 1.70GB / 4.07GB, 159MB/s + New Data Upload : 89%|████████▊ | 1.55GB / 1.74GB, 145MB/s  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 42%|████▏ | 1.72GB / 4.06GB  Processing Files (1 / 2) : 42%|████▏ | 1.73GB / 4.07GB, 162MB/s + New Data Upload : 91%|█████████ | 1.58GB / 1.74GB, 148MB/s  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 43%|████▎ | 1.75GB / 4.06GB  Processing Files (1 / 2) : 43%|████▎ | 1.76GB / 4.07GB, 164MB/s + New Data Upload : 92%|█████████▏| 1.61GB / 1.74GB, 150MB/s  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 44%|████▍ | 1.79GB / 4.06GB  Processing Files (1 / 2) : 44%|████▍ | 1.80GB / 4.07GB, 166MB/s + New Data Upload : 91%|█████████ | 1.65GB / 1.81GB, 152MB/s  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 45%|████▍ | 1.82GB / 4.06GB  Processing Files (1 / 2) : 45%|████▍ | 1.83GB / 4.07GB, 168MB/s + New Data Upload : 89%|████████▉ | 1.68GB / 1.88GB, 154MB/s  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 46%|████▌ | 1.85GB / 4.06GB  Processing Files (1 / 2) : 46%|████▌ | 1.86GB / 4.07GB, 169MB/s + New Data Upload : 91%|█████████ | 1.71GB / 1.88GB, 155MB/s  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 47%|████▋ | 1.90GB / 4.06GB  Processing Files (1 / 2) : 47%|████▋ | 1.91GB / 4.07GB, 170MB/s + New Data Upload : 93%|█████████▎| 1.76GB / 1.88GB, 156MB/s  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 47%|████▋ | 1.92GB / 4.06GB  Processing Files (1 / 2) : 47%|████▋ | 1.93GB / 4.07GB, 171MB/s + New Data Upload : 91%|█████████▏| 1.78GB / 1.94GB, 157MB/s  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 48%|████▊ | 1.94GB / 4.06GB  Processing Files (1 / 2) : 48%|████▊ | 1.95GB / 4.07GB, 171MB/s + New Data Upload : 93%|█████████▎| 1.80GB / 1.94GB, 157MB/s  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 49%|████▊ | 1.97GB / 4.06GB  Processing Files (1 / 2) : 49%|████▊ | 1.98GB / 4.07GB, 171MB/s + New Data Upload : 91%|█████████ | 1.83GB / 2.01GB, 157MB/s  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 49%|████▉ | 2.00GB / 4.06GB  Processing Files (1 / 2) : 49%|████▉ | 2.01GB / 4.07GB, 172MB/s + New Data Upload : 92%|█████████▏| 1.86GB / 2.01GB, 158MB/s  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 50%|████▉ | 2.02GB / 4.06GB  Processing Files (1 / 2) : 50%|████▉ | 2.03GB / 4.07GB, 172MB/s + New Data Upload : 90%|█████████ | 1.88GB / 2.08GB, 158MB/s  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 51%|█████ | 2.06GB / 4.06GB  Processing Files (1 / 2) : 51%|█████ | 2.07GB / 4.07GB, 173MB/s + New Data Upload : 89%|████████▉ | 1.91GB / 2.15GB, 159MB/s  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 51%|█████▏ | 2.09GB / 4.06GB  Processing Files (1 / 2) : 52%|█████▏ | 2.10GB / 4.07GB, 174MB/s + New Data Upload : 91%|█████████ | 1.95GB / 2.15GB, 160MB/s  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 53%|█████▎ | 2.14GB / 4.06GB  Processing Files (1 / 2) : 53%|█████▎ | 2.15GB / 4.07GB, 175MB/s + New Data Upload : 90%|█████████ | 2.00GB / 2.21GB, 161MB/s  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 54%|█████▎ | 2.18GB / 4.06GB  Processing Files (1 / 2) : 54%|█████▍ | 2.19GB / 4.07GB, 176MB/s + New Data Upload : 89%|████████▉ | 2.04GB / 2.28GB, 163MB/s  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 54%|█████▍ | 2.21GB / 4.06GB  Processing Files (1 / 2) : 55%|█████▍ | 2.23GB / 4.07GB, 176MB/s + New Data Upload : 91%|█████████ | 2.07GB / 2.28GB, 162MB/s  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 55%|█████▌ | 2.25GB / 4.06GB  Processing Files (1 / 2) : 56%|█████▌ | 2.26GB / 4.07GB, 177MB/s + New Data Upload : 90%|████████▉ | 2.11GB / 2.35GB, 163MB/s  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 56%|█████▌ | 2.28GB / 4.06GB  Processing Files (1 / 2) : 56%|█████▌ | 2.29GB / 4.07GB, 176MB/s + New Data Upload : 91%|█████████ | 2.13GB / 2.35GB, 163MB/s  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 57%|█████▋ | 2.31GB / 4.06GB  Processing Files (1 / 2) : 57%|█████▋ | 2.32GB / 4.07GB, 176MB/s + New Data Upload : 90%|████████▉ | 2.17GB / 2.41GB, 162MB/s  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 58%|█████▊ | 2.34GB / 4.06GB  Processing Files (1 / 2) : 58%|█████▊ | 2.35GB / 4.07GB, 176MB/s + New Data Upload : 91%|█████████ | 2.20GB / 2.41GB, 162MB/s  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 58%|█████▊ | 2.38GB / 4.06GB  Processing Files (1 / 2) : 59%|█████▊ | 2.39GB / 4.07GB, 177MB/s + New Data Upload : 90%|█████████ | 2.24GB / 2.48GB, 163MB/s  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 59%|█████▉ | 2.41GB / 4.06GB  Processing Files (1 / 2) : 59%|█████▉ | 2.42GB / 4.07GB, 178MB/s + New Data Upload : 91%|█████████▏| 2.27GB / 2.48GB, 165MB/s  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 60%|██████ | 2.45GB / 4.06GB  Processing Files (1 / 2) : 60%|██████ | 2.46GB / 4.07GB, 180MB/s + New Data Upload : 93%|█████████▎| 2.30GB / 2.48GB, 166MB/s  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 61%|██████ | 2.48GB / 4.06GB  Processing Files (1 / 2) : 61%|██████ | 2.49GB / 4.07GB, 180MB/s + New Data Upload : 92%|█████████▏| 2.33GB / 2.55GB, 166MB/s  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 62%|██████▏ | 2.52GB / 4.06GB  Processing Files (1 / 2) : 62%|██████▏ | 2.53GB / 4.07GB, 180MB/s + New Data Upload : 91%|█████████ | 2.37GB / 2.62GB, 166MB/s  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 63%|██████▎ | 2.56GB / 4.06GB  Processing Files (1 / 2) : 63%|██████▎ | 2.57GB / 4.07GB, 182MB/s + New Data Upload : 92%|█████████▏| 2.42GB / 2.62GB, 168MB/s  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 64%|██████▍ | 2.59GB / 4.06GB  Processing Files (1 / 2) : 64%|██████▍ | 2.61GB / 4.07GB, 182MB/s + New Data Upload : 91%|█████████▏| 2.45GB / 2.68GB, 168MB/s  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 65%|██████▍ | 2.64GB / 4.06GB  Processing Files (1 / 2) : 65%|██████▌ | 2.65GB / 4.07GB, 183MB/s + New Data Upload : 91%|█████████ | 2.50GB / 2.75GB, 169MB/s  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 66%|██████▌ | 2.67GB / 4.06GB  Processing Files (1 / 2) : 66%|██████▌ | 2.68GB / 4.07GB, 183MB/s + New Data Upload : 92%|█████████▏| 2.53GB / 2.75GB, 169MB/s  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 67%|██████▋ | 2.72GB / 4.06GB  Processing Files (1 / 2) : 67%|██████▋ | 2.73GB / 4.07GB, 185MB/s + New Data Upload : 91%|█████████▏| 2.57GB / 2.82GB, 172MB/s  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 68%|██████▊ | 2.75GB / 4.06GB  Processing Files (1 / 2) : 68%|██████▊ | 2.76GB / 4.07GB, 181MB/s + New Data Upload : 93%|█████████▎| 2.61GB / 2.82GB, 171MB/s  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 68%|██████▊ | 2.78GB / 4.06GB  Processing Files (1 / 2) : 68%|██████▊ | 2.79GB / 4.07GB, 177MB/s + New Data Upload : 91%|█████████▏| 2.63GB / 2.88GB, 170MB/s  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 69%|██████▉ | 2.80GB / 4.06GB  Processing Files (1 / 2) : 69%|██████▉ | 2.82GB / 4.07GB, 171MB/s + New Data Upload : 92%|█████████▏| 2.66GB / 2.88GB, 170MB/s  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 70%|███████ | 2.85GB / 4.06GB  Processing Files (1 / 2) : 70%|███████ | 2.86GB / 4.07GB, 171MB/s + New Data Upload : 92%|█████████▏| 2.71GB / 2.95GB, 171MB/s  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 71%|███████▏ | 2.90GB / 4.06GB  Processing Files (1 / 2) : 71%|███████▏ | 2.91GB / 4.07GB, 172MB/s + New Data Upload : 91%|█████████▏| 2.75GB / 3.02GB, 172MB/s  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 72%|███████▏ | 2.93GB / 4.06GB  Processing Files (1 / 2) : 72%|███████▏ | 2.94GB / 4.07GB, 171MB/s + New Data Upload : 92%|█████████▏| 2.79GB / 3.02GB, 171MB/s  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 73%|███████▎ | 2.98GB / 4.06GB  Processing Files (1 / 2) : 73%|███████▎ | 2.99GB / 4.07GB, 174MB/s + New Data Upload : 92%|█████████▏| 2.84GB / 3.09GB, 174MB/s  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 74%|███████▍ | 3.02GB / 4.06GB  Processing Files (1 / 2) : 74%|███████▍ | 3.03GB / 4.07GB, 176MB/s + New Data Upload : 93%|█████████▎| 2.88GB / 3.09GB, 176MB/s  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 75%|███████▌ | 3.07GB / 4.06GB  Processing Files (1 / 2) : 76%|███████▌ | 3.08GB / 4.07GB, 178MB/s + New Data Upload : 93%|█████████▎| 2.93GB / 3.15GB, 178MB/s  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 76%|███████▋ | 3.10GB / 4.06GB  Processing Files (1 / 2) : 76%|███████▋ | 3.12GB / 4.07GB, 180MB/s + New Data Upload : 92%|█████████▏| 2.96GB / 3.22GB, 180MB/s  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 77%|███████▋ | 3.13GB / 4.06GB  Processing Files (1 / 2) : 77%|███████▋ | 3.14GB / 4.07GB, 179MB/s + New Data Upload : 93%|█████████▎| 2.99GB / 3.22GB, 179MB/s  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 78%|███████▊ | 3.17GB / 4.06GB  Processing Files (1 / 2) : 78%|███████▊ | 3.18GB / 4.07GB, 178MB/s + New Data Upload : 92%|█████████▏| 3.03GB / 3.29GB, 178MB/s  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 79%|███████▉ | 3.20GB / 4.06GB  Processing Files (1 / 2) : 79%|███████▉ | 3.21GB / 4.07GB, 177MB/s + New Data Upload : 93%|█████████▎| 3.06GB / 3.29GB, 177MB/s  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 80%|███████▉ | 3.24GB / 4.06GB  Processing Files (1 / 2) : 80%|███████▉ | 3.26GB / 4.07GB, 177MB/s + New Data Upload : 93%|█████████▎| 3.10GB / 3.35GB, 177MB/s  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 81%|████████ | 3.29GB / 4.06GB  Processing Files (1 / 2) : 81%|████████ | 3.30GB / 4.07GB, 178MB/s + New Data Upload : 94%|█████████▍| 3.15GB / 3.35GB, 178MB/s  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 82%|████████▏ | 3.32GB / 4.06GB  Processing Files (1 / 2) : 82%|████████▏ | 3.33GB / 4.07GB, 178MB/s + New Data Upload : 93%|█████████▎| 3.18GB / 3.42GB, 178MB/s  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 83%|████████▎ | 3.36GB / 4.06GB  Processing Files (1 / 2) : 83%|████████▎ | 3.37GB / 4.07GB, 179MB/s + New Data Upload : 92%|█████████▏| 3.22GB / 3.49GB, 179MB/s  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 84%|████████▍ | 3.41GB / 4.06GB  Processing Files (1 / 2) : 84%|████████▍ | 3.42GB / 4.07GB, 180MB/s + New Data Upload : 94%|█████████▎| 3.27GB / 3.49GB, 180MB/s  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 85%|████████▌ | 3.46GB / 4.06GB  Processing Files (1 / 2) : 85%|████████▌ | 3.47GB / 4.07GB, 181MB/s + New Data Upload : 95%|█████████▌| 3.32GB / 3.49GB, 181MB/s  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 86%|████████▌ | 3.50GB / 4.06GB  Processing Files (1 / 2) : 86%|████████▌ | 3.51GB / 4.07GB, 181MB/s + New Data Upload : 94%|█████████▍| 3.36GB / 3.55GB, 181MB/s  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 87%|████████▋ | 3.55GB / 4.06GB  Processing Files (1 / 2) : 87%|████████▋ | 3.56GB / 4.07GB, 182MB/s + New Data Upload : 94%|█████████▍| 3.40GB / 3.62GB, 182MB/s  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 88%|████████▊ | 3.58GB / 4.06GB  Processing Files (1 / 2) : 88%|████████▊ | 3.59GB / 4.07GB, 183MB/s + New Data Upload : 95%|█████████▌| 3.44GB / 3.62GB, 183MB/s  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 89%|████████▉ | 3.62GB / 4.06GB  Processing Files (1 / 2) : 89%|████████▉ | 3.63GB / 4.07GB, 183MB/s + New Data Upload : 94%|█████████▍| 3.48GB / 3.69GB, 183MB/s  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 90%|████████▉ | 3.65GB / 4.06GB  Processing Files (1 / 2) : 90%|████████▉ | 3.66GB / 4.07GB, 182MB/s + New Data Upload : 95%|█████████▌| 3.51GB / 3.69GB, 182MB/s  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 91%|█████████ | 3.68GB / 4.06GB  Processing Files (1 / 2) : 91%|█████████ | 3.69GB / 4.07GB, 182MB/s + New Data Upload : 94%|█████████▍| 3.54GB / 3.76GB, 182MB/s  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 91%|█████████▏| 3.71GB / 4.06GB  Processing Files (1 / 2) : 91%|█████████▏| 3.72GB / 4.07GB, 182MB/s + New Data Upload : 95%|█████████▍| 3.57GB / 3.76GB, 182MB/s  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 92%|█████████▏| 3.74GB / 4.06GB  Processing Files (1 / 2) : 92%|█████████▏| 3.75GB / 4.07GB, 181MB/s + New Data Upload : 94%|█████████▍| 3.60GB / 3.82GB, 181MB/s  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 93%|█████████▎| 3.78GB / 4.06GB  Processing Files (1 / 2) : 93%|█████████▎| 3.79GB / 4.07GB, 182MB/s + New Data Upload : 95%|█████████▌| 3.64GB / 3.82GB, 182MB/s  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 94%|█████████▎| 3.81GB / 4.06GB  Processing Files (1 / 2) : 94%|█████████▍| 3.82GB / 4.07GB, 183MB/s + New Data Upload : 94%|█████████▍| 3.67GB / 3.89GB, 183MB/s  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 95%|█████████▍| 3.86GB / 4.06GB  Processing Files (1 / 2) : 95%|█████████▍| 3.87GB / 4.07GB, 185MB/s + New Data Upload : 96%|█████████▌| 3.72GB / 3.89GB, 185MB/s  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 96%|█████████▌| 3.91GB / 4.06GB  Processing Files (1 / 2) : 96%|█████████▌| 3.92GB / 4.07GB, 187MB/s + New Data Upload : 97%|█████████▋| 3.76GB / 3.89GB, 187MB/s  + + + + ...ing/fft/training_args.bin: 13%|█▎ | 937B / 7.18kB  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 97%|█████████▋| 3.94GB / 4.06GB  + + + + ...ing/fft/training_args.bin: 13%|█▎ | 937B / 7.18kB  Processing Files (1 / 3) : 97%|█████████▋| 3.96GB / 4.07GB, 189MB/s + New Data Upload : 97%|█████████▋| 3.80GB / 3.92GB, 189MB/s  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 98%|█████████▊| 3.96GB / 4.06GB  + + + + ...ing/fft/training_args.bin: 19%|█▊ | 1.34kB / 7.18kB  Processing Files (1 / 3) : 98%|█████████▊| 3.97GB / 4.07GB, 187MB/s + New Data Upload : 97%|█████████▋| 3.82GB / 3.92GB, 187MB/s  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 98%|█████████▊| 3.99GB / 4.06GB  + + + + ...ing/fft/training_args.bin: 39%|███▉ | 2.81kB / 7.18kB  Processing Files (1 / 3) : 98%|█████████▊| 4.00GB / 4.07GB, 186MB/s + New Data Upload : 98%|█████████▊| 3.85GB / 3.92GB, 186MB/s  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 99%|█████████▊| 4.01GB / 4.06GB  + + + + ...ing/fft/training_args.bin: 58%|█████▊ | 4.15kB / 7.18kB  Processing Files (1 / 3) : 99%|█████████▊| 4.02GB / 4.07GB, 183MB/s + New Data Upload : 99%|█████████▊| 3.87GB / 3.92GB, 183MB/s  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 99%|█████████▉| 4.02GB / 4.06GB  + + + + ...ing/fft/training_args.bin: 78%|███████▊ | 5.62kB / 7.18kB  Processing Files (1 / 3) : 99%|█████████▉| 4.03GB / 4.07GB, 181MB/s + New Data Upload : 99%|█████████▉| 3.88GB / 3.92GB, 181MB/s  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 99%|█████████▉| 4.04GB / 4.06GB  + + + + ...ing/fft/training_args.bin: 99%|█████████▊| 7.09kB / 7.18kB  Processing Files (1 / 3) : 99%|█████████▉| 4.05GB / 4.07GB, 179MB/s + New Data Upload : 99%|█████████▉| 3.90GB / 3.92GB, 179MB/s  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 100%|█████████▉| 4.04GB / 4.06GB  + + + + ...ing/fft/training_args.bin: 99%|█████████▊| 7.09kB / 7.18kB  Processing Files (1 / 3) : 100%|█████████▉| 4.06GB / 4.07GB, 176MB/s + New Data Upload : 100%|█████████▉| 3.90GB / 3.92GB, 176MB/s  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 100%|█████████▉| 4.05GB / 4.06GB  + + + + ...ing/fft/training_args.bin: 99%|█████████▊| 7.09kB / 7.18kB  Processing Files (1 / 3) : 100%|█████████▉| 4.06GB / 4.07GB, 174MB/s + New Data Upload : 100%|█████████▉| 3.91GB / 3.92GB, 174MB/s  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 100%|█████████▉| 4.06GB / 4.06GB  + + + + ...ing/fft/training_args.bin: 100%|██████████| 7.18kB / 7.18kB  Processing Files (2 / 3) : 100%|█████████▉| 4.07GB / 4.07GB, 172MB/s + New Data Upload : 100%|█████████▉| 3.92GB / 3.92GB, 172MB/s  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 100%|█████████▉| 4.06GB / 4.06GB  + + + + ...ing/fft/training_args.bin: 100%|██████████| 7.18kB / 7.18kB  Processing Files (2 / 3) : 100%|█████████▉| 4.07GB / 4.07GB, 169MB/s + New Data Upload : 100%|█████████▉| 3.92GB / 3.92GB, 169MB/s  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 100%|█████████▉| 4.06GB / 4.06GB  + + + + ...ing/fft/training_args.bin: 100%|██████████| 7.18kB / 7.18kB  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 100%|█████████▉| 4.06GB / 4.06GB  + + + + ...ing/fft/training_args.bin: 100%|██████████| 7.18kB / 7.18kB  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 100%|█████████▉| 4.06GB / 4.06GB  + + + + ...ing/fft/training_args.bin: 100%|██████████| 7.18kB / 7.18kB  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 100%|█████████▉| 4.06GB / 4.06GB  + + + + ...ing/fft/training_args.bin: 100%|██████████| 7.18kB / 7.18kB  Processing Files (2 / 3) : 100%|█████████▉| 4.07GB / 4.07GB, 156MB/s + New Data Upload : 100%|█████████▉| 3.92GB / 3.92GB, 156MB/s  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 100%|█████████▉| 4.06GB / 4.06GB  + + + + ...ing/fft/training_args.bin: 100%|██████████| 7.18kB / 7.18kB  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 100%|█████████▉| 4.06GB / 4.06GB  + + + + ...ing/fft/training_args.bin: 100%|██████████| 7.18kB / 7.18kB  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 100%|█████████▉| 4.06GB / 4.06GB  + + + + ...ing/fft/training_args.bin: 100%|██████████| 7.18kB / 7.18kB  Processing Files (2 / 3) : 100%|█████████▉| 4.07GB / 4.07GB, 144MB/s + New Data Upload : 100%|██████████| 3.92GB / 3.92GB, 144MB/s  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 100%|█████████▉| 4.06GB / 4.06GB  + + + + ...ing/fft/training_args.bin: 100%|██████████| 7.18kB / 7.18kB  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 100%|█████████▉| 4.06GB / 4.06GB  + + + + ...ing/fft/training_args.bin: 100%|██████████| 7.18kB / 7.18kB  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 100%|█████████▉| 4.06GB / 4.06GB  + + + + ...ing/fft/training_args.bin: 100%|██████████| 7.18kB / 7.18kB  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 100%|█████████▉| 4.06GB / 4.06GB  + + + + ...ing/fft/training_args.bin: 100%|██████████| 7.18kB / 7.18kB  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 100%|█████████▉| 4.06GB / 4.06GB  + + + + ...ing/fft/training_args.bin: 100%|██████████| 7.18kB / 7.18kB  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 100%|█████████▉| 4.06GB / 4.06GB  + + + + ...ing/fft/training_args.bin: 100%|██████████| 7.18kB / 7.18kB  + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + ...ing/fft/model.safetensors: 100%|█████████▉| 4.06GB / 4.06GB  + + + + ...ing/fft/training_args.bin: 100%|██████████| 7.18kB / 7.18kB  Processing Files (2 / 3) : 100%|█████████▉| 4.07GB / 4.07GB, 122MB/s + New Data Upload : 100%|██████████| 3.92GB / 3.92GB, 122MB/s + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB + ...ing/fft/model.safetensors: 100%|█████████▉| 4.06GB / 4.06GB + ...ing/fft/training_args.bin: 100%|██████████| 7.18kB / 7.18kB +***** train metrics ***** + epoch = 4.0 + total_flos = 742719798GF + train_loss = 0.6604 + train_runtime = 0:52:35.39 + train_samples_per_second = 38.03 + train_steps_per_second = 0.298 +Figure saved at: saves/qwen3_1.7b/coding/fft/training_loss.png +[WARNING|2026-04-29 17:59:55] llamafactory.extras.ploting:149 >> No metric eval_loss to plot. +[WARNING|2026-04-29 17:59:55] llamafactory.extras.ploting:149 >> No metric eval_accuracy to plot. +[INFO|trainer.py:3797] 2026-04-29 17:59:57,214 >> Saving model checkpoint to saves/qwen3_1.7b/coding/fft +[INFO|configuration_utils.py:432] 2026-04-29 17:59:57,221 >> Configuration saved in saves/qwen3_1.7b/coding/fft/config.json +[INFO|configuration_utils.py:803] 2026-04-29 17:59:57,226 >> Configuration saved in saves/qwen3_1.7b/coding/fft/generation_config.json + Writing model shards: 0%| | 0/1 [00:00> Model weights saved in saves/qwen3_1.7b/coding/fft/model.safetensors +[INFO|tokenization_utils_base.py:3224] 2026-04-29 18:00:04,084 >> chat template saved in saves/qwen3_1.7b/coding/fft/chat_template.jinja +[INFO|tokenization_utils_base.py:2078] 2026-04-29 18:00:04,090 >> tokenizer config file saved in saves/qwen3_1.7b/coding/fft/tokenizer_config.json +[INFO|modelcard.py:266] 2026-04-29 18:00:04,389 >> Dropping the following result as it does not have all the necessary fields: +{'task': {'name': 'Causal Language Modeling', 'type': 'text-generation'}} + Processing Files (0 / 0) : | | 0.00B / 0.00B + New Data Upload : | | 0.00B / 0.00B  + + ...ing/fft/training_args.bin: 100%|██████████| 7.18kB / 7.18kB  + + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + + ...ing/fft/model.safetensors: 2%|▏ | 72.0MB / 4.06GB  + + ...ing/fft/training_args.bin: 100%|██████████| 7.18kB / 7.18kB  + + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + + ...ing/fft/model.safetensors: 2%|▏ | 72.0MB / 4.06GB  Processing Files (2 / 3) : 2%|▏ | 83.4MB / 4.07GB, ???B/s + + ...ing/fft/training_args.bin: 100%|██████████| 7.18kB / 7.18kB  + + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + + ...ing/fft/model.safetensors: 5%|▍ | 184MB / 4.06GB  Processing Files (2 / 3) : 5%|▍ | 195MB / 4.07GB, 560MB/s + + ...ing/fft/training_args.bin: 100%|██████████| 7.18kB / 7.18kB  + + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + + ...ing/fft/model.safetensors: 8%|▊ | 336MB / 4.06GB  Processing Files (2 / 3) : 9%|▊ | 347MB / 4.07GB, 661MB/s + + ...ing/fft/training_args.bin: 100%|██████████| 7.18kB / 7.18kB  + + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + + ...ing/fft/model.safetensors: 12%|█▏ | 496MB / 4.06GB  Processing Files (2 / 3) : 12%|█▏ | 507MB / 4.07GB, 707MB/s + + ...ing/fft/training_args.bin: 100%|██████████| 7.18kB / 7.18kB  + + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + + ...ing/fft/model.safetensors: 16%|█▌ | 656MB / 4.06GB  Processing Files (2 / 3) : 16%|█▋ | 667MB / 4.07GB, 731MB/s + + ...ing/fft/training_args.bin: 100%|██████████| 7.18kB / 7.18kB  + + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + + ...ing/fft/model.safetensors: 20%|██ | 824MB / 4.06GB  Processing Files (2 / 3) : 21%|██ | 835MB / 4.07GB, 753MB/s + + ...ing/fft/training_args.bin: 100%|██████████| 7.18kB / 7.18kB  + + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + + ...ing/fft/model.safetensors: 24%|██▍ | 992MB / 4.06GB  Processing Files (2 / 3) : 25%|██▍ | 1.00GB / 4.07GB, 767MB/s + + ...ing/fft/training_args.bin: 100%|██████████| 7.18kB / 7.18kB  + + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + + ...ing/fft/model.safetensors: 28%|██▊ | 1.14GB / 4.06GB  Processing Files (2 / 3) : 28%|██▊ | 1.15GB / 4.07GB, 762MB/s + + ...ing/fft/training_args.bin: 100%|██████████| 7.18kB / 7.18kB  + + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + + ...ing/fft/model.safetensors: 31%|███▏ | 1.28GB / 4.06GB  Processing Files (2 / 3) : 32%|███▏ | 1.29GB / 4.07GB, 754MB/s + + ...ing/fft/training_args.bin: 100%|██████████| 7.18kB / 7.18kB  + + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + + ...ing/fft/model.safetensors: 36%|███▌ | 1.45GB / 4.06GB  Processing Files (2 / 3) : 36%|███▌ | 1.46GB / 4.07GB, 763MB/s + + ...ing/fft/training_args.bin: 100%|██████████| 7.18kB / 7.18kB  + + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + + ...ing/fft/model.safetensors: 40%|███▉ | 1.61GB / 4.06GB  Processing Files (2 / 3) : 40%|███▉ | 1.63GB / 4.07GB, 771MB/s + + ...ing/fft/training_args.bin: 100%|██████████| 7.18kB / 7.18kB  + + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + + ...ing/fft/model.safetensors: 44%|████▍ | 1.78GB / 4.06GB  Processing Files (2 / 3) : 44%|████▍ | 1.79GB / 4.07GB, 778MB/s + + ...ing/fft/training_args.bin: 100%|██████████| 7.18kB / 7.18kB  + + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + + ...ing/fft/model.safetensors: 48%|████▊ | 1.95GB / 4.06GB  Processing Files (2 / 3) : 48%|████▊ | 1.96GB / 4.07GB, 783MB/s + + ...ing/fft/training_args.bin: 100%|██████████| 7.18kB / 7.18kB  + + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + + ...ing/fft/model.safetensors: 52%|█████▏ | 2.12GB / 4.06GB  Processing Files (2 / 3) : 52%|█████▏ | 2.13GB / 4.07GB, 787MB/s + + ...ing/fft/training_args.bin: 100%|██████████| 7.18kB / 7.18kB  + + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + + ...ing/fft/model.safetensors: 56%|█████▋ | 2.29GB / 4.06GB  Processing Files (2 / 3) : 56%|█████▋ | 2.30GB / 4.07GB, 791MB/s + + ...ing/fft/training_args.bin: 100%|██████████| 7.18kB / 7.18kB  + + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + + ...ing/fft/model.safetensors: 60%|██████ | 2.45GB / 4.06GB  Processing Files (2 / 3) : 61%|██████ | 2.47GB / 4.07GB, 794MB/s + + ...ing/fft/training_args.bin: 100%|██████████| 7.18kB / 7.18kB  + + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + + ...ing/fft/model.safetensors: 65%|██████▍ | 2.62GB / 4.06GB  Processing Files (2 / 3) : 65%|██████▍ | 2.63GB / 4.07GB, 797MB/s + + ...ing/fft/training_args.bin: 100%|██████████| 7.18kB / 7.18kB  + + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + + ...ing/fft/model.safetensors: 68%|██████▊ | 2.78GB / 4.06GB  Processing Files (2 / 3) : 69%|██████▊ | 2.79GB / 4.07GB, 797MB/s + + ...ing/fft/training_args.bin: 100%|██████████| 7.18kB / 7.18kB  + + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + + ...ing/fft/model.safetensors: 73%|███████▎ | 2.96GB / 4.06GB  Processing Files (2 / 3) : 73%|███████▎ | 2.97GB / 4.07GB, 802MB/s + + ...ing/fft/training_args.bin: 100%|██████████| 7.18kB / 7.18kB  + + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + + ...ing/fft/model.safetensors: 77%|███████▋ | 3.13GB / 4.06GB  Processing Files (2 / 3) : 77%|███████▋ | 3.14GB / 4.07GB, 804MB/s + + ...ing/fft/training_args.bin: 100%|██████████| 7.18kB / 7.18kB  + + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + + ...ing/fft/model.safetensors: 81%|████████ | 3.29GB / 4.06GB  Processing Files (2 / 3) : 81%|████████ | 3.31GB / 4.07GB, 806MB/s + + ...ing/fft/training_args.bin: 100%|██████████| 7.18kB / 7.18kB  + + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + + ...ing/fft/model.safetensors: 85%|████████▌ | 3.46GB / 4.06GB  Processing Files (2 / 3) : 85%|████████▌ | 3.47GB / 4.07GB, 807MB/s + + ...ing/fft/training_args.bin: 100%|██████████| 7.18kB / 7.18kB  + + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + + ...ing/fft/model.safetensors: 89%|████████▉ | 3.63GB / 4.06GB  Processing Files (2 / 3) : 89%|████████▉ | 3.64GB / 4.07GB, 809MB/s + + ...ing/fft/training_args.bin: 100%|██████████| 7.18kB / 7.18kB  + + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + + ...ing/fft/model.safetensors: 93%|█████████▎| 3.80GB / 4.06GB  Processing Files (2 / 3) : 93%|█████████▎| 3.81GB / 4.07GB, 810MB/s + + ...ing/fft/training_args.bin: 100%|██████████| 7.18kB / 7.18kB  + + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + + ...ing/fft/model.safetensors: 98%|█████████▊| 3.97GB / 4.06GB  Processing Files (2 / 3) : 98%|█████████▊| 3.98GB / 4.07GB, 811MB/s + + ...ing/fft/training_args.bin: 100%|██████████| 7.18kB / 7.18kB  + + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + + ...ing/fft/model.safetensors: 100%|█████████▉| 4.06GB / 4.06GB  Processing Files (2 / 3) : 100%|█████████▉| 4.07GB / 4.07GB, 797MB/s + New Data Upload : 43%|████▎ | 4.32MB / 10.1MB, 864kB/s  + + ...ing/fft/training_args.bin: 100%|██████████| 7.18kB / 7.18kB  + + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + + ...ing/fft/model.safetensors: 100%|█████████▉| 4.06GB / 4.06GB  + + ...ing/fft/training_args.bin: 100%|██████████| 7.18kB / 7.18kB  + + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + + ...ing/fft/model.safetensors: 100%|█████████▉| 4.06GB / 4.06GB  + + ...ing/fft/training_args.bin: 100%|██████████| 7.18kB / 7.18kB  + + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + + ...ing/fft/model.safetensors: 100%|█████████▉| 4.06GB / 4.06GB  Processing Files (2 / 3) : 100%|█████████▉| 4.07GB / 4.07GB, 713MB/s + New Data Upload : 98%|█████████▊| 9.87MB / 10.1MB, 1.76MB/s  + + ...ing/fft/training_args.bin: 100%|██████████| 7.18kB / 7.18kB  + + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + + ...ing/fft/model.safetensors: 100%|█████████▉| 4.06GB / 4.06GB  + + ...ing/fft/training_args.bin: 100%|██████████| 7.18kB / 7.18kB  + + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + + ...ing/fft/model.safetensors: 100%|██████████| 4.06GB / 4.06GB  Processing Files (3 / 3) : 100%|██████████| 4.07GB / 4.07GB, 665MB/s + New Data Upload : 100%|██████████| 10.1MB / 10.1MB, 1.68MB/s  + + ...ing/fft/training_args.bin: 100%|██████████| 7.18kB / 7.18kB  + + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + + ...ing/fft/model.safetensors: 100%|██████████| 4.06GB / 4.06GB  + + ...ing/fft/training_args.bin: 100%|██████████| 7.18kB / 7.18kB  + + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + + ...ing/fft/model.safetensors: 100%|██████████| 4.06GB / 4.06GB  + + ...ing/fft/training_args.bin: 100%|██████████| 7.18kB / 7.18kB  + + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + + ...ing/fft/model.safetensors: 100%|██████████| 4.06GB / 4.06GB  + + ...ing/fft/training_args.bin: 100%|██████████| 7.18kB / 7.18kB  + + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + + ...ing/fft/model.safetensors: 100%|██████████| 4.06GB / 4.06GB  + + ...ing/fft/training_args.bin: 100%|██████████| 7.18kB / 7.18kB  + + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + + ...ing/fft/model.safetensors: 100%|██████████| 4.06GB / 4.06GB  + + ...ing/fft/training_args.bin: 100%|██████████| 7.18kB / 7.18kB  + + + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB  + + + + ...ing/fft/model.safetensors: 100%|██████████| 4.06GB / 4.06GB  Processing Files (3 / 3) : 100%|██████████| 4.07GB / 4.07GB, 570MB/s + New Data Upload : 100%|██████████| 10.1MB / 10.1MB, 1.44MB/s + ...ing/fft/training_args.bin: 100%|██████████| 7.18kB / 7.18kB + ...coding/fft/tokenizer.json: 100%|██████████| 11.4MB / 11.4MB + ...ing/fft/model.safetensors: 100%|██████████| 4.06GB / 4.06GB diff --git a/tokenizer.json b/tokenizer.json new file mode 100644 index 0000000..c7afbed --- /dev/null +++ b/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:be75606093db2094d7cd20f3c2f385c212750648bd6ea4fb2bf507a6a4c55506 +size 11422650 diff --git a/tokenizer_config.json b/tokenizer_config.json new file mode 100644 index 0000000..145e2c7 --- /dev/null +++ b/tokenizer_config.json @@ -0,0 +1,30 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|im_end|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "padding_side": "right", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/train_results.json b/train_results.json new file mode 100644 index 0000000..d751c7c --- /dev/null +++ b/train_results.json @@ -0,0 +1,8 @@ +{ + "epoch": 4.0, + "total_flos": 7.974893111623025e+17, + "train_loss": 0.6604051219656112, + "train_runtime": 3155.399, + "train_samples_per_second": 38.03, + "train_steps_per_second": 0.298 +} \ No newline at end of file diff --git a/trainer_log.jsonl b/trainer_log.jsonl new file mode 100644 index 0000000..80a5f50 --- /dev/null +++ b/trainer_log.jsonl @@ -0,0 +1,95 @@ +{"current_steps": 10, "total_steps": 940, "loss": 1.291497802734375, "lr": 9.574468085106384e-07, "epoch": 0.042666666666666665, "percentage": 1.06, "elapsed_time": "0:00:35", "remaining_time": "0:55:03"} +{"current_steps": 20, "total_steps": 940, "loss": 1.182865810394287, "lr": 2.021276595744681e-06, "epoch": 0.08533333333333333, "percentage": 2.13, "elapsed_time": "0:01:10", "remaining_time": "0:53:55"} +{"current_steps": 30, "total_steps": 940, "loss": 0.9575789451599122, "lr": 3.0851063829787237e-06, "epoch": 0.128, "percentage": 3.19, "elapsed_time": "0:01:43", "remaining_time": "0:52:21"} +{"current_steps": 40, "total_steps": 940, "loss": 0.8388131141662598, "lr": 4.148936170212766e-06, "epoch": 0.17066666666666666, "percentage": 4.26, "elapsed_time": "0:02:13", "remaining_time": "0:49:54"} +{"current_steps": 50, "total_steps": 940, "loss": 0.8148186683654786, "lr": 5.212765957446809e-06, "epoch": 0.21333333333333335, "percentage": 5.32, "elapsed_time": "0:02:50", "remaining_time": "0:50:42"} +{"current_steps": 60, "total_steps": 940, "loss": 0.7651220321655273, "lr": 6.276595744680851e-06, "epoch": 0.256, "percentage": 6.38, "elapsed_time": "0:03:22", "remaining_time": "0:49:25"} +{"current_steps": 70, "total_steps": 940, "loss": 0.7899383544921875, "lr": 7.340425531914894e-06, "epoch": 0.2986666666666667, "percentage": 7.45, "elapsed_time": "0:03:57", "remaining_time": "0:49:15"} +{"current_steps": 80, "total_steps": 940, "loss": 0.7782301902770996, "lr": 8.404255319148937e-06, "epoch": 0.3413333333333333, "percentage": 8.51, "elapsed_time": "0:04:30", "remaining_time": "0:48:25"} +{"current_steps": 90, "total_steps": 940, "loss": 0.7684392929077148, "lr": 9.46808510638298e-06, "epoch": 0.384, "percentage": 9.57, "elapsed_time": "0:05:07", "remaining_time": "0:48:22"} +{"current_steps": 100, "total_steps": 940, "loss": 0.7520565032958985, "lr": 9.999138160172624e-06, "epoch": 0.4266666666666667, "percentage": 10.64, "elapsed_time": "0:05:40", "remaining_time": "0:47:40"} +{"current_steps": 110, "total_steps": 940, "loss": 0.7115557670593262, "lr": 9.99224522409411e-06, "epoch": 0.4693333333333333, "percentage": 11.7, "elapsed_time": "0:06:18", "remaining_time": "0:47:35"} +{"current_steps": 120, "total_steps": 940, "loss": 0.7434285163879395, "lr": 9.97846885608885e-06, "epoch": 0.512, "percentage": 12.77, "elapsed_time": "0:06:51", "remaining_time": "0:46:54"} +{"current_steps": 130, "total_steps": 940, "loss": 0.7457537651062012, "lr": 9.957828051355817e-06, "epoch": 0.5546666666666666, "percentage": 13.83, "elapsed_time": "0:07:29", "remaining_time": "0:46:41"} +{"current_steps": 140, "total_steps": 940, "loss": 0.7388599872589111, "lr": 9.930351269950144e-06, "epoch": 0.5973333333333334, "percentage": 14.89, "elapsed_time": "0:08:05", "remaining_time": "0:46:11"} +{"current_steps": 150, "total_steps": 940, "loss": 0.7285007476806641, "lr": 9.896076397541676e-06, "epoch": 0.64, "percentage": 15.96, "elapsed_time": "0:08:36", "remaining_time": "0:45:21"} +{"current_steps": 160, "total_steps": 940, "loss": 0.7659842491149902, "lr": 9.855050693177286e-06, "epoch": 0.6826666666666666, "percentage": 17.02, "elapsed_time": "0:09:08", "remaining_time": "0:44:33"} +{"current_steps": 170, "total_steps": 940, "loss": 0.724073839187622, "lr": 9.807330724118906e-06, "epoch": 0.7253333333333334, "percentage": 18.09, "elapsed_time": "0:09:45", "remaining_time": "0:44:11"} +{"current_steps": 180, "total_steps": 940, "loss": 0.747947883605957, "lr": 9.752982287847193e-06, "epoch": 0.768, "percentage": 19.15, "elapsed_time": "0:10:17", "remaining_time": "0:43:26"} +{"current_steps": 190, "total_steps": 940, "loss": 0.7283991813659668, "lr": 9.692080321338317e-06, "epoch": 0.8106666666666666, "percentage": 20.21, "elapsed_time": "0:10:48", "remaining_time": "0:42:41"} +{"current_steps": 200, "total_steps": 940, "loss": 0.7407590389251709, "lr": 9.624708797739002e-06, "epoch": 0.8533333333333334, "percentage": 21.28, "elapsed_time": "0:11:16", "remaining_time": "0:41:43"} +{"current_steps": 210, "total_steps": 940, "loss": 0.724196481704712, "lr": 9.550960610582251e-06, "epoch": 0.896, "percentage": 22.34, "elapsed_time": "0:11:50", "remaining_time": "0:41:11"} +{"current_steps": 220, "total_steps": 940, "loss": 0.7334166526794433, "lr": 9.47093744570344e-06, "epoch": 0.9386666666666666, "percentage": 23.4, "elapsed_time": "0:12:20", "remaining_time": "0:40:22"} +{"current_steps": 230, "total_steps": 940, "loss": 0.8169802665710449, "lr": 9.384749641033358e-06, "epoch": 0.9813333333333333, "percentage": 24.47, "elapsed_time": "0:12:49", "remaining_time": "0:39:34"} +{"current_steps": 240, "total_steps": 940, "loss": 0.7211981773376465, "lr": 9.292516034461517e-06, "epoch": 1.0213333333333334, "percentage": 25.53, "elapsed_time": "0:13:20", "remaining_time": "0:38:54"} +{"current_steps": 250, "total_steps": 940, "loss": 0.6335953712463379, "lr": 9.194363799979517e-06, "epoch": 1.064, "percentage": 26.6, "elapsed_time": "0:13:52", "remaining_time": "0:38:17"} +{"current_steps": 260, "total_steps": 940, "loss": 0.6689927577972412, "lr": 9.090428272330381e-06, "epoch": 1.1066666666666667, "percentage": 27.66, "elapsed_time": "0:14:23", "remaining_time": "0:37:38"} +{"current_steps": 270, "total_steps": 940, "loss": 0.6809295654296875, "lr": 8.980852760405645e-06, "epoch": 1.1493333333333333, "percentage": 28.72, "elapsed_time": "0:14:59", "remaining_time": "0:37:10"} +{"current_steps": 280, "total_steps": 940, "loss": 0.6914172172546387, "lr": 8.865788349647496e-06, "epoch": 1.192, "percentage": 29.79, "elapsed_time": "0:15:27", "remaining_time": "0:36:25"} +{"current_steps": 290, "total_steps": 940, "loss": 0.6650928974151611, "lr": 8.745393693728395e-06, "epoch": 1.2346666666666666, "percentage": 30.85, "elapsed_time": "0:16:00", "remaining_time": "0:35:53"} +{"current_steps": 300, "total_steps": 940, "loss": 0.6717124938964844, "lr": 8.619834795795458e-06, "epoch": 1.2773333333333334, "percentage": 31.91, "elapsed_time": "0:16:30", "remaining_time": "0:35:12"} +{"current_steps": 310, "total_steps": 940, "loss": 0.6726479530334473, "lr": 8.489284779581179e-06, "epoch": 1.32, "percentage": 32.98, "elapsed_time": "0:17:06", "remaining_time": "0:34:46"} +{"current_steps": 320, "total_steps": 940, "loss": 0.6797503471374512, "lr": 8.353923650696119e-06, "epoch": 1.3626666666666667, "percentage": 34.04, "elapsed_time": "0:17:37", "remaining_time": "0:34:08"} +{"current_steps": 330, "total_steps": 940, "loss": 0.6944191932678223, "lr": 8.213938048432697e-06, "epoch": 1.4053333333333333, "percentage": 35.11, "elapsed_time": "0:18:12", "remaining_time": "0:33:39"} +{"current_steps": 340, "total_steps": 940, "loss": 0.619914960861206, "lr": 8.069520988422292e-06, "epoch": 1.448, "percentage": 36.17, "elapsed_time": "0:18:46", "remaining_time": "0:33:07"} +{"current_steps": 350, "total_steps": 940, "loss": 0.6835647583007812, "lr": 7.920871596500473e-06, "epoch": 1.4906666666666666, "percentage": 37.23, "elapsed_time": "0:19:17", "remaining_time": "0:32:30"} +{"current_steps": 360, "total_steps": 940, "loss": 0.6706738471984863, "lr": 7.768194834147362e-06, "epoch": 1.5333333333333332, "percentage": 38.3, "elapsed_time": "0:19:49", "remaining_time": "0:31:55"} +{"current_steps": 370, "total_steps": 940, "loss": 0.690638542175293, "lr": 7.611701215881635e-06, "epoch": 1.576, "percentage": 39.36, "elapsed_time": "0:20:27", "remaining_time": "0:31:31"} +{"current_steps": 380, "total_steps": 940, "loss": 0.6714137077331543, "lr": 7.4516065189978625e-06, "epoch": 1.6186666666666667, "percentage": 40.43, "elapsed_time": "0:21:01", "remaining_time": "0:30:59"} +{"current_steps": 390, "total_steps": 940, "loss": 0.6779637336730957, "lr": 7.288131486047414e-06, "epoch": 1.6613333333333333, "percentage": 41.49, "elapsed_time": "0:21:36", "remaining_time": "0:30:28"} +{"current_steps": 400, "total_steps": 940, "loss": 0.6494068622589111, "lr": 7.121501520473137e-06, "epoch": 1.704, "percentage": 42.55, "elapsed_time": "0:22:13", "remaining_time": "0:30:00"} +{"current_steps": 410, "total_steps": 940, "loss": 0.6467916011810303, "lr": 6.9519463758174745e-06, "epoch": 1.7466666666666666, "percentage": 43.62, "elapsed_time": "0:22:48", "remaining_time": "0:29:29"} +{"current_steps": 420, "total_steps": 940, "loss": 0.6865511894226074, "lr": 6.77969983893257e-06, "epoch": 1.7893333333333334, "percentage": 44.68, "elapsed_time": "0:23:22", "remaining_time": "0:28:56"} +{"current_steps": 430, "total_steps": 940, "loss": 0.6830032348632813, "lr": 6.604999407629137e-06, "epoch": 1.8319999999999999, "percentage": 45.74, "elapsed_time": "0:23:57", "remaining_time": "0:28:25"} +{"current_steps": 440, "total_steps": 940, "loss": 0.6937445640563965, "lr": 6.428085963208567e-06, "epoch": 1.8746666666666667, "percentage": 46.81, "elapsed_time": "0:24:32", "remaining_time": "0:27:53"} +{"current_steps": 450, "total_steps": 940, "loss": 0.6877000331878662, "lr": 6.249203438329799e-06, "epoch": 1.9173333333333333, "percentage": 47.87, "elapsed_time": "0:25:08", "remaining_time": "0:27:22"} +{"current_steps": 460, "total_steps": 940, "loss": 0.6529797077178955, "lr": 6.0685984806689055e-06, "epoch": 1.96, "percentage": 48.94, "elapsed_time": "0:25:43", "remaining_time": "0:26:50"} +{"current_steps": 470, "total_steps": 940, "loss": 0.6860092163085938, "lr": 5.886520112835128e-06, "epoch": 2.0, "percentage": 50.0, "elapsed_time": "0:26:15", "remaining_time": "0:26:15"} +{"current_steps": 480, "total_steps": 940, "loss": 0.6075316905975342, "lr": 5.703219389012317e-06, "epoch": 2.042666666666667, "percentage": 51.06, "elapsed_time": "0:26:48", "remaining_time": "0:25:41"} +{"current_steps": 490, "total_steps": 940, "loss": 0.5943183422088623, "lr": 5.518949048799176e-06, "epoch": 2.0853333333333333, "percentage": 52.13, "elapsed_time": "0:27:20", "remaining_time": "0:25:07"} +{"current_steps": 500, "total_steps": 940, "loss": 0.5897200107574463, "lr": 5.3339631687256085e-06, "epoch": 2.128, "percentage": 53.19, "elapsed_time": "0:27:53", "remaining_time": "0:24:32"} +{"current_steps": 510, "total_steps": 940, "loss": 0.6106101036071777, "lr": 5.148516811925684e-06, "epoch": 2.1706666666666665, "percentage": 54.26, "elapsed_time": "0:28:36", "remaining_time": "0:24:07"} +{"current_steps": 520, "total_steps": 940, "loss": 0.6104888916015625, "lr": 4.962865676450239e-06, "epoch": 2.2133333333333334, "percentage": 55.32, "elapsed_time": "0:29:09", "remaining_time": "0:23:32"} +{"current_steps": 530, "total_steps": 940, "loss": 0.5798359870910644, "lr": 4.777265742704039e-06, "epoch": 2.2560000000000002, "percentage": 56.38, "elapsed_time": "0:29:42", "remaining_time": "0:22:59"} +{"current_steps": 540, "total_steps": 940, "loss": 0.6197110652923584, "lr": 4.591972920493638e-06, "epoch": 2.2986666666666666, "percentage": 57.45, "elapsed_time": "0:30:17", "remaining_time": "0:22:26"} +{"current_steps": 550, "total_steps": 940, "loss": 0.5831963539123535, "lr": 4.40724269617256e-06, "epoch": 2.3413333333333335, "percentage": 58.51, "elapsed_time": "0:30:50", "remaining_time": "0:21:52"} +{"current_steps": 560, "total_steps": 940, "loss": 0.598298454284668, "lr": 4.223329780370359e-06, "epoch": 2.384, "percentage": 59.57, "elapsed_time": "0:31:24", "remaining_time": "0:21:18"} +{"current_steps": 570, "total_steps": 940, "loss": 0.5884403705596923, "lr": 4.04048775679127e-06, "epoch": 2.4266666666666667, "percentage": 60.64, "elapsed_time": "0:32:00", "remaining_time": "0:20:46"} +{"current_steps": 580, "total_steps": 940, "loss": 0.5839417457580567, "lr": 3.858968732566685e-06, "epoch": 2.469333333333333, "percentage": 61.7, "elapsed_time": "0:32:34", "remaining_time": "0:20:13"} +{"current_steps": 590, "total_steps": 940, "loss": 0.6246652126312255, "lr": 3.6790229906435706e-06, "epoch": 2.512, "percentage": 62.77, "elapsed_time": "0:33:07", "remaining_time": "0:19:38"} +{"current_steps": 600, "total_steps": 940, "loss": 0.6042469024658204, "lr": 3.5008986446881088e-06, "epoch": 2.554666666666667, "percentage": 63.83, "elapsed_time": "0:33:48", "remaining_time": "0:19:09"} +{"current_steps": 610, "total_steps": 940, "loss": 0.6285554885864257, "lr": 3.3248412969804065e-06, "epoch": 2.5973333333333333, "percentage": 64.89, "elapsed_time": "0:34:21", "remaining_time": "0:18:35"} +{"current_steps": 620, "total_steps": 940, "loss": 0.6006599426269531, "lr": 3.1510936997719557e-06, "epoch": 2.64, "percentage": 65.96, "elapsed_time": "0:34:55", "remaining_time": "0:18:01"} +{"current_steps": 630, "total_steps": 940, "loss": 0.5928638458251954, "lr": 2.9798954205727886e-06, "epoch": 2.6826666666666665, "percentage": 67.02, "elapsed_time": "0:35:25", "remaining_time": "0:17:25"} +{"current_steps": 640, "total_steps": 940, "loss": 0.6103617668151855, "lr": 2.811482511829842e-06, "epoch": 2.7253333333333334, "percentage": 68.09, "elapsed_time": "0:35:58", "remaining_time": "0:16:51"} +{"current_steps": 650, "total_steps": 940, "loss": 0.5874536514282227, "lr": 2.6460871854519594e-06, "epoch": 2.768, "percentage": 69.15, "elapsed_time": "0:36:31", "remaining_time": "0:16:17"} +{"current_steps": 660, "total_steps": 940, "loss": 0.5812715053558349, "lr": 2.483937492630345e-06, "epoch": 2.8106666666666666, "percentage": 70.21, "elapsed_time": "0:37:04", "remaining_time": "0:15:43"} +{"current_steps": 670, "total_steps": 940, "loss": 0.609982681274414, "lr": 2.3252570093959e-06, "epoch": 2.8533333333333335, "percentage": 71.28, "elapsed_time": "0:37:37", "remaining_time": "0:15:09"} +{"current_steps": 680, "total_steps": 940, "loss": 0.6121101379394531, "lr": 2.1702645283470238e-06, "epoch": 2.896, "percentage": 72.34, "elapsed_time": "0:38:08", "remaining_time": "0:14:34"} +{"current_steps": 690, "total_steps": 940, "loss": 0.6171843528747558, "lr": 2.0191737569729492e-06, "epoch": 2.9386666666666668, "percentage": 73.4, "elapsed_time": "0:38:41", "remaining_time": "0:14:01"} +{"current_steps": 700, "total_steps": 940, "loss": 0.5880200386047363, "lr": 1.872193022988526e-06, "epoch": 2.981333333333333, "percentage": 74.47, "elapsed_time": "0:39:14", "remaining_time": "0:13:27"} +{"current_steps": 710, "total_steps": 940, "loss": 0.5795096397399903, "lr": 1.7295249870867898e-06, "epoch": 3.021333333333333, "percentage": 75.53, "elapsed_time": "0:39:46", "remaining_time": "0:12:53"} +{"current_steps": 720, "total_steps": 940, "loss": 0.5702518463134766, "lr": 1.5913663635053578e-06, "epoch": 3.064, "percentage": 76.6, "elapsed_time": "0:40:18", "remaining_time": "0:12:19"} +{"current_steps": 730, "total_steps": 940, "loss": 0.5839662551879883, "lr": 1.457907648791943e-06, "epoch": 3.1066666666666665, "percentage": 77.66, "elapsed_time": "0:40:51", "remaining_time": "0:11:45"} +{"current_steps": 740, "total_steps": 940, "loss": 0.5281820297241211, "lr": 1.329332859142967e-06, "epoch": 3.1493333333333333, "percentage": 78.72, "elapsed_time": "0:41:24", "remaining_time": "0:11:11"} +{"current_steps": 750, "total_steps": 940, "loss": 0.5751385688781738, "lr": 1.205819276677464e-06, "epoch": 3.192, "percentage": 79.79, "elapsed_time": "0:41:59", "remaining_time": "0:10:38"} +{"current_steps": 760, "total_steps": 940, "loss": 0.5494537353515625, "lr": 1.0875372049960697e-06, "epoch": 3.2346666666666666, "percentage": 80.85, "elapsed_time": "0:42:31", "remaining_time": "0:10:04"} +{"current_steps": 770, "total_steps": 940, "loss": 0.5537368297576905, "lr": 9.746497343621857e-07, "epoch": 3.2773333333333334, "percentage": 81.91, "elapsed_time": "0:43:06", "remaining_time": "0:09:31"} +{"current_steps": 780, "total_steps": 940, "loss": 0.55874662399292, "lr": 8.673125168290713e-07, "epoch": 3.32, "percentage": 82.98, "elapsed_time": "0:43:40", "remaining_time": "0:08:57"} +{"current_steps": 790, "total_steps": 940, "loss": 0.565284252166748, "lr": 7.656735516229125e-07, "epoch": 3.3626666666666667, "percentage": 84.04, "elapsed_time": "0:44:11", "remaining_time": "0:08:23"} +{"current_steps": 800, "total_steps": 940, "loss": 0.5531235694885254, "lr": 6.698729810778065e-07, "epoch": 3.405333333333333, "percentage": 85.11, "elapsed_time": "0:44:48", "remaining_time": "0:07:50"} +{"current_steps": 810, "total_steps": 940, "loss": 0.5778326511383056, "lr": 5.800428974040311e-07, "epoch": 3.448, "percentage": 86.17, "elapsed_time": "0:45:24", "remaining_time": "0:07:17"} +{"current_steps": 820, "total_steps": 940, "loss": 0.5518184661865234, "lr": 4.963071605560144e-07, "epoch": 3.490666666666667, "percentage": 87.23, "elapsed_time": "0:46:01", "remaining_time": "0:06:44"} +{"current_steps": 830, "total_steps": 940, "loss": 0.5773732185363769, "lr": 4.187812274511427e-07, "epoch": 3.533333333333333, "percentage": 88.3, "elapsed_time": "0:46:31", "remaining_time": "0:06:09"} +{"current_steps": 840, "total_steps": 940, "loss": 0.5663552284240723, "lr": 3.4757199277490106e-07, "epoch": 3.576, "percentage": 89.36, "elapsed_time": "0:47:03", "remaining_time": "0:05:36"} +{"current_steps": 850, "total_steps": 940, "loss": 0.5664983272552491, "lr": 2.8277764159181484e-07, "epoch": 3.618666666666667, "percentage": 90.43, "elapsed_time": "0:47:36", "remaining_time": "0:05:02"} +{"current_steps": 860, "total_steps": 940, "loss": 0.5846799373626709, "lr": 2.2448751396543788e-07, "epoch": 3.6613333333333333, "percentage": 91.49, "elapsed_time": "0:48:03", "remaining_time": "0:04:28"} +{"current_steps": 870, "total_steps": 940, "loss": 0.5573970794677734, "lr": 1.7278198177405614e-07, "epoch": 3.7039999999999997, "percentage": 92.55, "elapsed_time": "0:48:33", "remaining_time": "0:03:54"} +{"current_steps": 880, "total_steps": 940, "loss": 0.5604241371154786, "lr": 1.2773233789193816e-07, "epoch": 3.7466666666666666, "percentage": 93.62, "elapsed_time": "0:49:06", "remaining_time": "0:03:20"} +{"current_steps": 890, "total_steps": 940, "loss": 0.5811796188354492, "lr": 8.940069788894389e-08, "epoch": 3.7893333333333334, "percentage": 94.68, "elapsed_time": "0:49:39", "remaining_time": "0:02:47"} +{"current_steps": 900, "total_steps": 940, "loss": 0.5551981449127197, "lr": 5.783991438403802e-08, "epoch": 3.832, "percentage": 95.74, "elapsed_time": "0:50:13", "remaining_time": "0:02:13"} +{"current_steps": 910, "total_steps": 940, "loss": 0.5618691444396973, "lr": 3.309350417077972e-08, "epoch": 3.8746666666666667, "percentage": 96.81, "elapsed_time": "0:50:49", "remaining_time": "0:01:40"} +{"current_steps": 920, "total_steps": 940, "loss": 0.5391227722167968, "lr": 1.5195588215283773e-08, "epoch": 3.9173333333333336, "percentage": 97.87, "elapsed_time": "0:51:24", "remaining_time": "0:01:07"} +{"current_steps": 930, "total_steps": 940, "loss": 0.5671097278594971, "lr": 4.170844609387992e-09, "epoch": 3.96, "percentage": 98.94, "elapsed_time": "0:51:54", "remaining_time": "0:00:33"} +{"current_steps": 940, "total_steps": 940, "loss": 0.5210320949554443, "lr": 3.447454388127991e-11, "epoch": 4.0, "percentage": 100.0, "elapsed_time": "0:52:24", "remaining_time": "0:00:00"} +{"current_steps": 940, "total_steps": 940, "epoch": 4.0, "percentage": 100.0, "elapsed_time": "0:52:33", "remaining_time": "0:00:00"} diff --git a/trainer_state.json b/trainer_state.json new file mode 100644 index 0000000..66cfd9b --- /dev/null +++ b/trainer_state.json @@ -0,0 +1,701 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 4.0, + "eval_steps": 500, + "global_step": 940, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.042666666666666665, + "grad_norm": 22.81595230102539, + "learning_rate": 9.574468085106384e-07, + "loss": 1.291497802734375, + "step": 10 + }, + { + "epoch": 0.08533333333333333, + "grad_norm": 8.337886810302734, + "learning_rate": 2.021276595744681e-06, + "loss": 1.182865810394287, + "step": 20 + }, + { + "epoch": 0.128, + "grad_norm": 2.478001594543457, + "learning_rate": 3.0851063829787237e-06, + "loss": 0.9575789451599122, + "step": 30 + }, + { + "epoch": 0.17066666666666666, + "grad_norm": 1.4279086589813232, + "learning_rate": 4.148936170212766e-06, + "loss": 0.8388131141662598, + "step": 40 + }, + { + "epoch": 0.21333333333333335, + "grad_norm": 1.040152907371521, + "learning_rate": 5.212765957446809e-06, + "loss": 0.8148186683654786, + "step": 50 + }, + { + "epoch": 0.256, + "grad_norm": 0.9204385876655579, + "learning_rate": 6.276595744680851e-06, + "loss": 0.7651220321655273, + "step": 60 + }, + { + "epoch": 0.2986666666666667, + "grad_norm": 0.8933783769607544, + "learning_rate": 7.340425531914894e-06, + "loss": 0.7899383544921875, + "step": 70 + }, + { + "epoch": 0.3413333333333333, + "grad_norm": 0.926196277141571, + "learning_rate": 8.404255319148937e-06, + "loss": 0.7782301902770996, + "step": 80 + }, + { + "epoch": 0.384, + "grad_norm": 0.9980058073997498, + "learning_rate": 9.46808510638298e-06, + "loss": 0.7684392929077148, + "step": 90 + }, + { + "epoch": 0.4266666666666667, + "grad_norm": 0.9042799472808838, + "learning_rate": 9.999138160172624e-06, + "loss": 0.7520565032958985, + "step": 100 + }, + { + "epoch": 0.4693333333333333, + "grad_norm": 0.8850747346878052, + "learning_rate": 9.99224522409411e-06, + "loss": 0.7115557670593262, + "step": 110 + }, + { + "epoch": 0.512, + "grad_norm": 0.9096830487251282, + "learning_rate": 9.97846885608885e-06, + "loss": 0.7434285163879395, + "step": 120 + }, + { + "epoch": 0.5546666666666666, + "grad_norm": 0.940818190574646, + "learning_rate": 9.957828051355817e-06, + "loss": 0.7457537651062012, + "step": 130 + }, + { + "epoch": 0.5973333333333334, + "grad_norm": 0.9445077180862427, + "learning_rate": 9.930351269950144e-06, + "loss": 0.7388599872589111, + "step": 140 + }, + { + "epoch": 0.64, + "grad_norm": 0.8463093042373657, + "learning_rate": 9.896076397541676e-06, + "loss": 0.7285007476806641, + "step": 150 + }, + { + "epoch": 0.6826666666666666, + "grad_norm": 0.8920971751213074, + "learning_rate": 9.855050693177286e-06, + "loss": 0.7659842491149902, + "step": 160 + }, + { + "epoch": 0.7253333333333334, + "grad_norm": 0.9197255969047546, + "learning_rate": 9.807330724118906e-06, + "loss": 0.724073839187622, + "step": 170 + }, + { + "epoch": 0.768, + "grad_norm": 0.9320898652076721, + "learning_rate": 9.752982287847193e-06, + "loss": 0.747947883605957, + "step": 180 + }, + { + "epoch": 0.8106666666666666, + "grad_norm": 0.8373203277587891, + "learning_rate": 9.692080321338317e-06, + "loss": 0.7283991813659668, + "step": 190 + }, + { + "epoch": 0.8533333333333334, + "grad_norm": 0.8440167307853699, + "learning_rate": 9.624708797739002e-06, + "loss": 0.7407590389251709, + "step": 200 + }, + { + "epoch": 0.896, + "grad_norm": 0.9070562720298767, + "learning_rate": 9.550960610582251e-06, + "loss": 0.724196481704712, + "step": 210 + }, + { + "epoch": 0.9386666666666666, + "grad_norm": 0.8882600665092468, + "learning_rate": 9.47093744570344e-06, + "loss": 0.7334166526794433, + "step": 220 + }, + { + "epoch": 0.9813333333333333, + "grad_norm": 0.9043309688568115, + "learning_rate": 9.384749641033358e-06, + "loss": 0.8169802665710449, + "step": 230 + }, + { + "epoch": 1.0213333333333334, + "grad_norm": 0.8862775564193726, + "learning_rate": 9.292516034461517e-06, + "loss": 0.7211981773376465, + "step": 240 + }, + { + "epoch": 1.064, + "grad_norm": 0.8914813995361328, + "learning_rate": 9.194363799979517e-06, + "loss": 0.6335953712463379, + "step": 250 + }, + { + "epoch": 1.1066666666666667, + "grad_norm": 0.9117757081985474, + "learning_rate": 9.090428272330381e-06, + "loss": 0.6689927577972412, + "step": 260 + }, + { + "epoch": 1.1493333333333333, + "grad_norm": 0.9108397364616394, + "learning_rate": 8.980852760405645e-06, + "loss": 0.6809295654296875, + "step": 270 + }, + { + "epoch": 1.192, + "grad_norm": 0.9220501780509949, + "learning_rate": 8.865788349647496e-06, + "loss": 0.6914172172546387, + "step": 280 + }, + { + "epoch": 1.2346666666666666, + "grad_norm": 0.8409727215766907, + "learning_rate": 8.745393693728395e-06, + "loss": 0.6650928974151611, + "step": 290 + }, + { + "epoch": 1.2773333333333334, + "grad_norm": 0.9512217044830322, + "learning_rate": 8.619834795795458e-06, + "loss": 0.6717124938964844, + "step": 300 + }, + { + "epoch": 1.32, + "grad_norm": 0.8877426385879517, + "learning_rate": 8.489284779581179e-06, + "loss": 0.6726479530334473, + "step": 310 + }, + { + "epoch": 1.3626666666666667, + "grad_norm": 0.8496021032333374, + "learning_rate": 8.353923650696119e-06, + "loss": 0.6797503471374512, + "step": 320 + }, + { + "epoch": 1.4053333333333333, + "grad_norm": 0.8745173811912537, + "learning_rate": 8.213938048432697e-06, + "loss": 0.6944191932678223, + "step": 330 + }, + { + "epoch": 1.448, + "grad_norm": 0.8742923736572266, + "learning_rate": 8.069520988422292e-06, + "loss": 0.619914960861206, + "step": 340 + }, + { + "epoch": 1.4906666666666666, + "grad_norm": 0.9417562484741211, + "learning_rate": 7.920871596500473e-06, + "loss": 0.6835647583007812, + "step": 350 + }, + { + "epoch": 1.5333333333333332, + "grad_norm": 0.9726133942604065, + "learning_rate": 7.768194834147362e-06, + "loss": 0.6706738471984863, + "step": 360 + }, + { + "epoch": 1.576, + "grad_norm": 0.8959471583366394, + "learning_rate": 7.611701215881635e-06, + "loss": 0.690638542175293, + "step": 370 + }, + { + "epoch": 1.6186666666666667, + "grad_norm": 0.9309572577476501, + "learning_rate": 7.4516065189978625e-06, + "loss": 0.6714137077331543, + "step": 380 + }, + { + "epoch": 1.6613333333333333, + "grad_norm": 0.9652783274650574, + "learning_rate": 7.288131486047414e-06, + "loss": 0.6779637336730957, + "step": 390 + }, + { + "epoch": 1.704, + "grad_norm": 0.8535190224647522, + "learning_rate": 7.121501520473137e-06, + "loss": 0.6494068622589111, + "step": 400 + }, + { + "epoch": 1.7466666666666666, + "grad_norm": 0.9693210124969482, + "learning_rate": 6.9519463758174745e-06, + "loss": 0.6467916011810303, + "step": 410 + }, + { + "epoch": 1.7893333333333334, + "grad_norm": 0.913555383682251, + "learning_rate": 6.77969983893257e-06, + "loss": 0.6865511894226074, + "step": 420 + }, + { + "epoch": 1.8319999999999999, + "grad_norm": 0.8626614212989807, + "learning_rate": 6.604999407629137e-06, + "loss": 0.6830032348632813, + "step": 430 + }, + { + "epoch": 1.8746666666666667, + "grad_norm": 0.9277188777923584, + "learning_rate": 6.428085963208567e-06, + "loss": 0.6937445640563965, + "step": 440 + }, + { + "epoch": 1.9173333333333333, + "grad_norm": 0.9122670292854309, + "learning_rate": 6.249203438329799e-06, + "loss": 0.6877000331878662, + "step": 450 + }, + { + "epoch": 1.96, + "grad_norm": 0.8714756369590759, + "learning_rate": 6.0685984806689055e-06, + "loss": 0.6529797077178955, + "step": 460 + }, + { + "epoch": 2.0, + "grad_norm": 1.4163620471954346, + "learning_rate": 5.886520112835128e-06, + "loss": 0.6860092163085938, + "step": 470 + }, + { + "epoch": 2.042666666666667, + "grad_norm": 0.8940126299858093, + "learning_rate": 5.703219389012317e-06, + "loss": 0.6075316905975342, + "step": 480 + }, + { + "epoch": 2.0853333333333333, + "grad_norm": 0.9413266181945801, + "learning_rate": 5.518949048799176e-06, + "loss": 0.5943183422088623, + "step": 490 + }, + { + "epoch": 2.128, + "grad_norm": 0.932360827922821, + "learning_rate": 5.3339631687256085e-06, + "loss": 0.5897200107574463, + "step": 500 + }, + { + "epoch": 2.1706666666666665, + "grad_norm": 0.9619448184967041, + "learning_rate": 5.148516811925684e-06, + "loss": 0.6106101036071777, + "step": 510 + }, + { + "epoch": 2.2133333333333334, + "grad_norm": 0.9640690684318542, + "learning_rate": 4.962865676450239e-06, + "loss": 0.6104888916015625, + "step": 520 + }, + { + "epoch": 2.2560000000000002, + "grad_norm": 0.9159106612205505, + "learning_rate": 4.777265742704039e-06, + "loss": 0.5798359870910644, + "step": 530 + }, + { + "epoch": 2.2986666666666666, + "grad_norm": 0.9406466484069824, + "learning_rate": 4.591972920493638e-06, + "loss": 0.6197110652923584, + "step": 540 + }, + { + "epoch": 2.3413333333333335, + "grad_norm": 1.0240126848220825, + "learning_rate": 4.40724269617256e-06, + "loss": 0.5831963539123535, + "step": 550 + }, + { + "epoch": 2.384, + "grad_norm": 0.9565466046333313, + "learning_rate": 4.223329780370359e-06, + "loss": 0.598298454284668, + "step": 560 + }, + { + "epoch": 2.4266666666666667, + "grad_norm": 0.9118269085884094, + "learning_rate": 4.04048775679127e-06, + "loss": 0.5884403705596923, + "step": 570 + }, + { + "epoch": 2.469333333333333, + "grad_norm": 0.9704470038414001, + "learning_rate": 3.858968732566685e-06, + "loss": 0.5839417457580567, + "step": 580 + }, + { + "epoch": 2.512, + "grad_norm": 0.949148416519165, + "learning_rate": 3.6790229906435706e-06, + "loss": 0.6246652126312255, + "step": 590 + }, + { + "epoch": 2.554666666666667, + "grad_norm": 0.9721353054046631, + "learning_rate": 3.5008986446881088e-06, + "loss": 0.6042469024658204, + "step": 600 + }, + { + "epoch": 2.5973333333333333, + "grad_norm": 1.11699378490448, + "learning_rate": 3.3248412969804065e-06, + "loss": 0.6285554885864257, + "step": 610 + }, + { + "epoch": 2.64, + "grad_norm": 1.023742437362671, + "learning_rate": 3.1510936997719557e-06, + "loss": 0.6006599426269531, + "step": 620 + }, + { + "epoch": 2.6826666666666665, + "grad_norm": 1.056766390800476, + "learning_rate": 2.9798954205727886e-06, + "loss": 0.5928638458251954, + "step": 630 + }, + { + "epoch": 2.7253333333333334, + "grad_norm": 1.0377728939056396, + "learning_rate": 2.811482511829842e-06, + "loss": 0.6103617668151855, + "step": 640 + }, + { + "epoch": 2.768, + "grad_norm": 0.9824387431144714, + "learning_rate": 2.6460871854519594e-06, + "loss": 0.5874536514282227, + "step": 650 + }, + { + "epoch": 2.8106666666666666, + "grad_norm": 1.0173842906951904, + "learning_rate": 2.483937492630345e-06, + "loss": 0.5812715053558349, + "step": 660 + }, + { + "epoch": 2.8533333333333335, + "grad_norm": 1.1885449886322021, + "learning_rate": 2.3252570093959e-06, + "loss": 0.609982681274414, + "step": 670 + }, + { + "epoch": 2.896, + "grad_norm": 0.974206805229187, + "learning_rate": 2.1702645283470238e-06, + "loss": 0.6121101379394531, + "step": 680 + }, + { + "epoch": 2.9386666666666668, + "grad_norm": 0.9060782194137573, + "learning_rate": 2.0191737569729492e-06, + "loss": 0.6171843528747558, + "step": 690 + }, + { + "epoch": 2.981333333333333, + "grad_norm": 1.0292282104492188, + "learning_rate": 1.872193022988526e-06, + "loss": 0.5880200386047363, + "step": 700 + }, + { + "epoch": 3.021333333333333, + "grad_norm": 1.00603449344635, + "learning_rate": 1.7295249870867898e-06, + "loss": 0.5795096397399903, + "step": 710 + }, + { + "epoch": 3.064, + "grad_norm": 0.9750069379806519, + "learning_rate": 1.5913663635053578e-06, + "loss": 0.5702518463134766, + "step": 720 + }, + { + "epoch": 3.1066666666666665, + "grad_norm": 0.9798346161842346, + "learning_rate": 1.457907648791943e-06, + "loss": 0.5839662551879883, + "step": 730 + }, + { + "epoch": 3.1493333333333333, + "grad_norm": 1.0039501190185547, + "learning_rate": 1.329332859142967e-06, + "loss": 0.5281820297241211, + "step": 740 + }, + { + "epoch": 3.192, + "grad_norm": 0.9754037857055664, + "learning_rate": 1.205819276677464e-06, + "loss": 0.5751385688781738, + "step": 750 + }, + { + "epoch": 3.2346666666666666, + "grad_norm": 0.990886926651001, + "learning_rate": 1.0875372049960697e-06, + "loss": 0.5494537353515625, + "step": 760 + }, + { + "epoch": 3.2773333333333334, + "grad_norm": 0.9957782626152039, + "learning_rate": 9.746497343621857e-07, + "loss": 0.5537368297576905, + "step": 770 + }, + { + "epoch": 3.32, + "grad_norm": 1.0468347072601318, + "learning_rate": 8.673125168290713e-07, + "loss": 0.55874662399292, + "step": 780 + }, + { + "epoch": 3.3626666666666667, + "grad_norm": 0.949479877948761, + "learning_rate": 7.656735516229125e-07, + "loss": 0.565284252166748, + "step": 790 + }, + { + "epoch": 3.405333333333333, + "grad_norm": 1.3505475521087646, + "learning_rate": 6.698729810778065e-07, + "loss": 0.5531235694885254, + "step": 800 + }, + { + "epoch": 3.448, + "grad_norm": 1.0080723762512207, + "learning_rate": 5.800428974040311e-07, + "loss": 0.5778326511383056, + "step": 810 + }, + { + "epoch": 3.490666666666667, + "grad_norm": 0.9772297143936157, + "learning_rate": 4.963071605560144e-07, + "loss": 0.5518184661865234, + "step": 820 + }, + { + "epoch": 3.533333333333333, + "grad_norm": 1.0470722913742065, + "learning_rate": 4.187812274511427e-07, + "loss": 0.5773732185363769, + "step": 830 + }, + { + "epoch": 3.576, + "grad_norm": 0.9107851982116699, + "learning_rate": 3.4757199277490106e-07, + "loss": 0.5663552284240723, + "step": 840 + }, + { + "epoch": 3.618666666666667, + "grad_norm": 0.9870362281799316, + "learning_rate": 2.8277764159181484e-07, + "loss": 0.5664983272552491, + "step": 850 + }, + { + "epoch": 3.6613333333333333, + "grad_norm": 1.0563737154006958, + "learning_rate": 2.2448751396543788e-07, + "loss": 0.5846799373626709, + "step": 860 + }, + { + "epoch": 3.7039999999999997, + "grad_norm": 1.0350754261016846, + "learning_rate": 1.7278198177405614e-07, + "loss": 0.5573970794677734, + "step": 870 + }, + { + "epoch": 3.7466666666666666, + "grad_norm": 1.0310237407684326, + "learning_rate": 1.2773233789193816e-07, + "loss": 0.5604241371154786, + "step": 880 + }, + { + "epoch": 3.7893333333333334, + "grad_norm": 0.9566333889961243, + "learning_rate": 8.940069788894389e-08, + "loss": 0.5811796188354492, + "step": 890 + }, + { + "epoch": 3.832, + "grad_norm": 0.9202328324317932, + "learning_rate": 5.783991438403802e-08, + "loss": 0.5551981449127197, + "step": 900 + }, + { + "epoch": 3.8746666666666667, + "grad_norm": 0.9610781073570251, + "learning_rate": 3.309350417077972e-08, + "loss": 0.5618691444396973, + "step": 910 + }, + { + "epoch": 3.9173333333333336, + "grad_norm": 0.9684038162231445, + "learning_rate": 1.5195588215283773e-08, + "loss": 0.5391227722167968, + "step": 920 + }, + { + "epoch": 3.96, + "grad_norm": 0.9603227972984314, + "learning_rate": 4.170844609387992e-09, + "loss": 0.5671097278594971, + "step": 930 + }, + { + "epoch": 4.0, + "grad_norm": 1.651139497756958, + "learning_rate": 3.447454388127991e-11, + "loss": 0.5210320949554443, + "step": 940 + }, + { + "epoch": 4.0, + "step": 940, + "total_flos": 7.974893111623025e+17, + "train_loss": 0.6604051219656112, + "train_runtime": 3155.399, + "train_samples_per_second": 38.03, + "train_steps_per_second": 0.298 + } + ], + "logging_steps": 10, + "max_steps": 940, + "num_input_tokens_seen": 0, + "num_train_epochs": 4, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": true + }, + "attributes": {} + } + }, + "total_flos": 7.974893111623025e+17, + "train_batch_size": 2, + "trial_name": null, + "trial_params": null +} diff --git a/training_args.bin b/training_args.bin new file mode 100644 index 0000000..593c891 --- /dev/null +++ b/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:26266069f0a3a139205f0115ae3df892adf8170151f2b6bb7f88e40771227ec5 +size 7185 diff --git a/training_loss.png b/training_loss.png new file mode 100644 index 0000000000000000000000000000000000000000..987ffc59ea4d8ec30e426c0f86164131171957ad GIT binary patch literal 40381 zcmd?Rk_DBVamC@Kho0ZNB-NC;BWpwcDX-LZg0 z=R0qFp69&3^X2>j=fmFH?OJzSHRhOOj=A1GmX{$uLvsd&LJ`Z}mwbXk;rO6X=rKZk z_#6I?p(*%}u!EGkgOZK0gY#26Bb3}z2U|-U2TQYOmz<33?9FVfc{#YbID~FoGIemU zwHM~(wEEvSIBe`pIGLq8HQ+9%ZSQN?qfn$zk$=!>5~*e=R1B-EMy&CJZ4*vn?8{B|ke`jh0V zmtwA6yY1HYh-b6R`;qbe`#6`cMTG9#>)7iwNAQ}&HY6RmG#*~h$jBhP{EF>BC}@^uI9^QhPG$>pEar+tu5$aLPF zy8Z7)(m3(RCz4cG|9^9vW!(o4&Ik@WDCNhBJfDzcmXGdBST@VxHFI{i9jiif{G7Ye zY|I;wu(~@Nw_hroaN*~CN)#ap$(A#(y!mwF=j#DAji1>)>#TpyGe3Fq5d9n6uF7z@J~n4wO<$315-jG zgzT9`oL8piQ{!e1w?|EW`JX$oev)ZdIxHz8gI&sR+bd;l5wIB%Z57|QOh`=Rj(D`O zG8{#~;Nk;M>a6H-9`&rdSu|7=f5*4oydagrsq97i_rXpD3D1G$^yTf!V=-KOqFad; z4txcp`L*L?0~R^S#l_*?2a}9t$4C1Pn30%-ge%#lE5bN9IO@6jXiVj?_?AMN7nLDeZ|kOzJC2$+pdA>$ofIPRe#>=ulM}y4mW!*oIhU^E9UO9m{p{n zXLx$B$cE+m^|XfGy)j=>^&h6^Okx}C7&fp@AjqF7(W<86MGS{-r9<8}eH;(d+ zCr--)-BYuF{`^@V96c%=S!0Rw_%u@P&TBVS&#bp7IX^$YG+6XraKt6Czt{A%TX{u` z?&7cQ=(sr2uH#+HrD8JTkC*kIOUJrQoMv#FyNr6Gq~v)|G$JAb1^aKQKYthFecV*_ z0+*X6=!RAyks`VqC4N(wJ-=<+{P1Y*5y&RB`>$I#n(;RWn z!GRCfSle~_^X8kM^3k|+^pSgWiSJ$Z=2K-KJg70`d0vf2WxG07?UqxnUyb9oF?R=U zm?$0eY;&R8gfE9nOf1GCr~He*KOqh-uF(EczDAy*Y&llfb5jMMgzmn+{-yEp@pshR zFG($lO+P2kr z2M5@;in=%$^G|0CN!2}4u~!M#Q|XRGRNP4!#hYOZ$c3nEYc}OXxxU;vI<5?HXAlBF-IQq!a@`iNcM_9e}RuN3J(_q<`8yZhi zw-r=Wj<$*_KZb@%!Yj|lI5WU9TEt-L{{DUPL-S@%*?IuR6}GS`AI+7kR~1#doe$us zT;brzZnd!P%Y6~ZSwWrO{|K(V!2C#md7!|t)pg=63m||m0qLGwe~U41zHv*9c|ngm zBg?v%Q>uv6)ytQ^O5JPy6GSWIvU&;D-oW>LYHBL4>$)MF===BYkEN%^W?(n5ns6|T z^CRqAh3cG_<~kCG?3!*~yLL^Vk49EbZW>;r#=5At#7oI*E0|8G=7(7);h8g6n3>-Y zFnW+hwWwmsF3h5gjg1qopJ|gg7PYC^WfpVa`en+;V>fkQz-mAlv&-mxzzyh7Wyr&A z+I~qKR#AGEqQ|QZIpr`O6ap4fisYl1B#1vaTyP!CaY~dm;MonzvzYSjSj@J)Ol;H* z&EO4G0IyxQirRPYW3MAXowfJg2fk@(*HM63p0fw5wWJ;!9Wu?aVo~syqnyQIdR`Gt zoR!hEHf85ZhgV$_i9Y>dZx0}C<^G%o&QW#wOy{1$!rCZqsZs)<%Ro4tA$ zCA>txe*Jm?Ga#AMi_P&q419ZmWgQTi+ik-%`{x2%O-+ppyxne#jaO|~x{`vhY$faR z@DH;Ki#=JBuxnXZS>HA{KSBlrajAY~xb!p;ks(Bejkylr_u_l?+kVX!9SKq}E@an? z_O3&m*g7P*Dl8lc%lt;my5#lxx*dbd*bC+&;!A>pVOE2MYh6kTyTM{xeFg{1Hr@sc zUFnZvgg<9#71;rvcAusIR41b0^H<4KeGZ6@&NXhKA(#3Jkz%;idEbp(Ek_rbRnNA= zaX-dz9-}h|xUAJN$|QEI!d%)N?at=AZJJ95(WuN?*bY1PevFK4JlMs;A!k2^4e|Ev z+i?IZ_ZY0QN^)8nVQOas97dO^k1`OoytIg>H%E_;sN$FKp zVLo0`QBgr)?8E2JzhO;f?%u^gVw=Rl>b;QBSvV!P0hOGGo)bWnhY3V>@5k8Km9t$B zzub7bw=qw|AR4}lomG1Bqy_@)T|h10-fxGonBqy2^A|2e2-+}g4?8nnl?lEkAVBF; z;kmy)6J@gw(+op@&=xBe!Dn_J0uVc(=ET;Z4MDs~TdbUyM8XB7*SGGqiET%pNxOLQ z;wA&Ga*%Z55v=jc<@u}z_`3DE()>v1*XiMmgwvcqp8%hgR2aah*y6>v zhwM|!jFkr5Crq+3GJbj;x=}nn#>O4*1h>S9sC6%Hnm>R39TVPStW;Z5(-mb?(eJ$U zMax=oa&od8PKa=C8O(`Fe*cntdCCttc#Y+vQB_o|1whJs=^Cv+^Jh{%^slv>W1Z zW(bQyw!LR&cScG|N(Pm#%U4X_#19LJGL_|j)J-E?Vb=^l4ZAt52Qu~H@`rt}jG z>K!xWD49B6;J&@m;#JW(uWVpo(5;v#-R*7(!Idq31n{fEz`97QeCpFfW#8&}N^xbQ%D8Rz<@zi)JVVjFgLu zOUQt8Lh{2f9`mj=;d^Qj!iT)}mko`L^W3+st2#3srs_Z9`vsN3iuUC`#jOjbw^&~+ z)#j+!Ul9TRCHS0CX-;sg1g4*%;v^0O+a<_Aol&Z7X zSaDBb5M_f(HdvCI_Wo$=CNlNB=?Ajqj zFxG8>H6bA(?b|oe`F$HJcGYi}I5~4V#`WS`pIxY8ymGh{xtCw$F#lu%=4B?*!ebT>AAirSHS!stlB1VQ^iJtCpx@F^ zNytTB%gT|Y+G9or44@vkvJ(i868U#IL2CFKjFRu+$5;QICI;+y$j>)TV0hT8&V7hZ?{SB zsOBZ!+2ZLhS_{o_mKMi{81D}Af(kP4D?!pqvl2`w&%F(F8n1#w!zU&nwl#uN8jl~} zc6Al<#U~b?UliOl!K{wgZY*?X%Fhwdcvu@m?w2NiH(;qSxvUug8ODP7_K{1LZh0io z9=ec*UlPZ=3ST&M8rP?rkj+mMy)#Ufr8+pamLhumD0XmQZyVDFL1hQ1lhACm{mw?G z0`gy%gS9%(S#9TG$E4I$>_TSFrw<>h?AyfZV59Cr02(j?@Y&Xl_1ZH-k~;CdxvR)o z`@*RwZ)j*(9-BSinTiFFd!nxH-(oxoFEa%<+1mW&)^-fUpb1`wwk(KNFuTDDyX7Gt zBV3??k+^*MMOT-yYL@12h_i|+pScbGaKSXg`U=5LfsJDftg6CJdSrWb$i9t#k3l}# zxqk@LkK`hZjF>3j-X1{R9f-2wd}eRX0oy9l<~%g-@F^mq;lt&7k&bd0uBTRj z!&%r!6l64y;>2TMfXG2#$jC&}nFtZ*?3`Ef9v-5QDGMV1?C$PHJN9UZ9&U67kx)p< z$XuYN?lYwYPTA1V@DF?-)qMp)cfIB;=T5AWj?VB{V;3@Sz%JGSYtGZsJ_O{5S!e3# z=y>(&)tFt$LYJ_G^&lh_H9lzEg^eXQi9lQelCJp(w(HkTbNyTocGkXW76ia7f+XB-V^1y1QX>&`On_vVejyjXPl!omXc_P3%R5U)N31O$YT9RD1) z{@dB*jUb-b;kacgVoh#u`oz=kl~e*UNIw~<0>^DEU%;P=`bVHkDD6iU-oC7%5vL})RR zPYcIXOd_WsitpFZhjgXXyyuhBJvn+95##UbF4z`9Sy|b&mXI5Oe^^mCNHFflDinUT zFPf$^w(CS85rLV(Z&GnFE6_yblnP*V!nNiwZyrE?qam$%_2y0N*1oWcdwGuMGt&CZ zoC?iZ?DoQFr38SPF%aAvnm_z^c97FNB#fDaZ8`%i+G>^5d-=(2IkUmP$qyF~U&H{- zax0Hj)AKBlFV%CKPsXprpR9Y&!pj>X6-c@J<`q8;u|Zw%G(^r{d!fj$~_P_@TVKyzhH&c-vEezp4WQGmxTS z54BWOblsQvH{8Uiv~AI2yV1`K3>M2sgQROA2it+S>??D%FSl@$D23&a1Ck9-?`D-T zD$sO=SyR|FGR=GS>J&aP%`fjGY@Bf1p@PKWuhtlm#puCyckAHz_)GoivRzljd?wxz zGm6vs08q4PdmnBhngZw@AuGbRzGF<$~T~AT)%hPk#k`PD;9lXdeLMAdAgFNHgh7 zk#ltGqggMiJPZPyh83Q*g*ctqIlt3edGxx)*kX&)G}WB5DmhHI5iWS&Z)ptM=9^w+ zED-b9`uckO93hD44DL(ZkC-nM4cR>`kI|2Xe9!A(O$Mk!3owsAu+Oe@a2P|b9*@?c$= zE7qe9NVs;z?hLJ-*ey<2K9~+0kRoaTNhA0zC?RE`6LSlNB>7`|`(p_0nVoxdBy_is zkP!}C1mf1c#12GxIKmtsy$=l~U#UDk>ijtpOfRyt-o%MkXj9hExahq4Tp<7OdeSB&T3{XP5|s89^^P-`nVBPvVspC8Bm_TtU8~|p} zZOWGc)1riEf_e>s12pWo_RrFBZOmQ`B-5Py&+aLk(|!ZVl*%e9dalzH2!+n*FjbZb zqDjuyDOIHx_b7IQT^+Y>VfU;QXagy|aFVWl3%|L#dbl}HL>EwybUjFnW9*MMbG*}M zW8Jf9E?!Kh36d^=^lAabqNFdMKc{Cbum-NT-y=;2m^HtD-=Bt@JSinPIsN!xRzjhl zSw*EA=uZaFe^p;!-|nia+nZ(BR;Au7t=_TTL{=kx`EFhK&Zkf39btaVm6bzj=;*%5 z$BJq|Hf-aocNIPac;9IZ|E7_jQr+9q)TD$dnPuP?64F328*S}qH^@BGAR9Lh8Qg-& zO*hUjkxysj3~Ndjha}b9!=sFbj4WP0`j+BwfrVP4Rq)%lifDdL&NgSumPmfK_+Unf zit24d9ox*v%w8|Zftgxx?A1xlR{vh(9K3Ck;3Pz@B6egycBsLA{rY2A0;dAKN-y)r zk3X1mQFkHf4M=Xc%Vz~%Z|@Ngn^$^cwjbQn3jxLH9$puc2zB&=`kaDH4TG@5{qnhl zz%G~xegOehPlRMlYk+925+kpcD*DjMkCai>q&fVnw2VxqbfV&NN1~q_9G@=M&92Zm zF-!qR<$)bP7uVyUfPg1(n7thP^Ni#;a68-Eh~mW|k8uq4+!^;Xuk<=pPV7h{q2arK z2uGwF!TjakoKe?H`H?UI-2^~# zj@Z6&ATxtH)8P;C)eYMN@O%7NTYG6+heEO`{MPd;NK#!Hby*%x(K|I=*W}Jqm;GuL zO{}?H3eyj!!}L5VwS))VDVN`GAUFBUuC@+(wH@FhD6S1i9ETKY9^#i)p=G+#eN6{? z`f>s;LBZ{$_XMX--Jdfx0_pZqB;VUn5FJ6v%XP-+j<`$`L#3hAY1!Pt`Wx9zS;`lL z9sU3&pF%=Fo>c>^W4?0bB@q49X5O~@I~V9xJv>HTFXdvi2hYZhO)BqhYuef=d^pMT}H1G(TO;H>vvB@%1Jeq!2LieUU%j_ zxQjD(an;)Xb4<+eH!)?T#?-OZ*wiE|FTZt2j|sSjibZ2`ZA?wmPRdOQzVnTpML%b% zivE?HnE&@W|E*lr(;;Oko!E?wTQTy$O)8_VX!-zAN$qr;7xGFyd4I_;d%0->jo93&eLQy&_4@nKg7^u9IHc`5ZN6245 z-K0DV*fQjSZP9k@R!J3=xPILqLMyLfJsD)&NGglm8z?D~4_mcAgCQNkJ3lWShAaqx zcn&k;PkEk<%ttq87V?I^Wv55XA(c^GD-MyUTSkrK5QzRxVf~0{|KvWc_1Fd#O1;d5 z8>sddWS(^1ZIC&NuSdj2dpMd~4i*MEF7_Y+!CO!YS-b=l$6Zu-Xy~gBQ$#m}xeyzI z8c9&BMSlNv$VTliz{(2;u*AB&glb7_k3FI@;*yGvZ9DmY={9@g6J-Hfkg}@kI*=qR zP=)qPHBK*V)F_|J3WQaJt_~KzMoNZFAj2M2f$r|E*`@(62sMn)n%54Jn+U3a>!R>U z86N5CGAM|yNhYFgSB6R;xYaq2mPR;O*p{mB(P+G7enmFXoSqTTG<^j!%(1btPB&}F zRoM$3>?nmX;$*ihI~nBn(|8V<@|~xl(VXp}>0!8TIDuu?C}dAn$>xN?PUiPp4gs_O z_3M{DPsA-qlKMbQMX{@8ODd8h@g2fhFw6wt>gF5W7%w*$RXQd)W)vMXNunPy?5 zE_pf}d0E+~P{2VJ6}W4AmX_cM{88A-9iR*7Kk17XFAxc~@?e_t7F}@9ft`P6ml=+} zkr5Lp1t$Aj%P2^(ay|E*^78UP9vKFa$77aLJ=Z6vVlM#V1yWfY@!GeF^*(a-0$Q+_ zxa)X;O}%GGq2)T&8DmKG@adDwuS_4xhYfNTb#B&49DM|C7z&g79>UdMSOaE-*hh$L z7A?cz`pP8M^$ke+T!!@@+ZNWa<5R_I(v3ihSGgyLzX8ATIzgjRmn3DrEOj zh?2rlwpt^dgXFn9&MVs4W$O=62>FJ@*{!iR4yYZH(ZT!d!(>&1>~$YXLueO>3!bwn zNlB}I5(h6xB@RAF9Id^VbMsEvbcE9d^kj@J&c)Ag;WIn53wo@wRi` zvcOC!!Btoewe$#QQ?nWEW=$X0Sn2h^#6X(6@_2_zx7;iI2GO~@{;Elx=Ec^MpGI{{ zlkxEI3U>HkxMjYikXn2FXLGUFelkdK9gr@q6zh2phB~4-bcJwrQgLnV@43Y@Le{mX6&gd zr~0(cvE@11o}1AoIcp9Dl_hL3)1}yhJ|5Tx%Xc0>CIk6`id1Za47jNQDBMF%gH}+K zu`ZeZ!mKw7@~P3$!LA4JK%_hhnL->eG4kZyL^(iiW_I>KWJzHT*BaS#R-pjIzwegd zcpqiyM=rItiK8l{tu1upZPcF>)avi8L@!Gir2p1Lgh}i;l0rIH?$aaXWdc%$sX?1c zCQ;F{hF%vq;V1PNII`5d#zdfFK8h8+2<6}>*Xqjg{EPS*cR>_Av z^!h@Bs=R*)8z*O-rGS#EYR^Q7R^=goSi%F5Nq3o#bm!`OzMaqBn-#wMNVxE5`tC*+ zmzpGaFd`Qg%pua;my;7qZ12}-P`Bp8Jqx7$n{GHfxRY@=1 z)_g+!@eV#)MvP_ajWv@_#M5yeXxU6w(a5voqeDavM2_~mcSP33E@j2- zl;52}^_;rxb9S$8b-w57x4w|JI8L(xHclkMa6|NUR6QSCB$d$qZFuu?` zsdv=!>F=v#x%v!W^_{y&|Ee=9&)oY(WhBeV;~&9&jD^2a#`oeKTD*C&UC)(^Tr3|4 z`}D?Ry-ISTs@+4;+A4{arn>1bLE3Q94?Uy;3hAuH+9HQ`@rT^pT*^vo;6S&RhXPbPR<`#S`PlJ`j@A*HDE9=pZ8W;t!h9>`Uy1{Qn*r z0uOCgBieh#aQjvTjrt7w%w@^mjRuxDE|*n(Fktw-e+$Wi@+f|!7d zrpCIRfr=SqHxL?6f$&lW#URJ*BKukSR*7Se(M==~WMhACBRu9*>bj7gdLN4TQ0aA; zdkh)njJqg5KZVcJ+BaXXpD<8(&(L?P}I)TsBd7*ua==$7?u4$?hpu0Cei#(U&$gH4W60 zs4R=bY)@UV{QKNh^KN-vj;kkq6lvJ%b|wL@r(qGLw>{Rov0kxWhkNHPT&T@Z z$wb+fXoHG)1S*|9xN5MF8?6DO35ki31CT{M4C9#2YvP!MY5w)+&!_P4dQj48UigQuI|;q;E7isMzT%aii|wX-9)OL8Mn+IwB?Vq{ z|NgC`B5T;ANbwbi@qZv#YdW9X{h zyzxOxC%YBC9t^*H_6}?ncG-syCtsfB&*-x~P*`Lk((W}#*O6P9Z@@D4??`#-M$fQ8Fv-pNt1EmqE; z%XUNCx@_B$oa)YQSR=Hcs7i6 zGC@?jp2vq813uFq2o zSWu^e%8KxqR=em|(w5=T2%fxB{6?hj-9gEp7MH$jnE zg91zp2(#a9M#ULir_L4HjJ87DA-(=^#ko^&fyZWY5BmPfT{fJi*=YQpOoABRzfOBfb|9idwZO(!Hu= z$Z%o-jYNX=EIm%PxgjBSFez@+gMVLs4PVZ7BGorobd*kdg}eIDNh8r7@5E19hCE^f zHK$rVV_E#n-CUVWu^NAgLr~)6r8J(&81Qi-R7J24<#RBy;4}L@3Dvq|M3!(%Qf$2LktISZ0h z1K4(0By)gd%Tjlyg^YSpgv~ikl6a`V?7BS#R zZZ6k47K>K6V<@F~FvRwpSy|<`dBLZ+I0bce_432luVqqC`#>r2#}7Vhkng~-lc5w_ z3GV@^$laNiC>eEiYN+9Jgy}e4q!qB#hqAU+`$WoFE>qU;!ZbJWaO<2;^?%ZM*_4&g zUC+q{ixp5@d<&&VNFH+!L`+OR#Yqxk)EL@1fHMnHc!s?Co3W*N?YH4xSyjhD|H|F$yzP5g2FO!Ilf0{lGkhi29*{0^h~ zD<5bOix?mjQdbO$<@G-DXg1Ere2I=8VzxTV)pY|$Na`(>jo5#d>lGQxWi%O3Vp11S z0(QYT=>PVu*+yB3{p=G})tOQw6B9r;9CJ5R=2B}&tRT^s1;jf^Dqar}gFi&0yNn*IXwb^XkW?80!qm*nr>Llh8XEh9vb9OQ z@z>)s9xsc!a*l;FeDwL}n&!&}p{XTGkk_)0OYd|T00bBO2`OEmKy&hVN;U?z!(a0b zcA%2NqG3vhy^jyQ%#Itb2Rt!R7z+^^q(T|}J9IQXW-};Ig4dE>m;l>yT*=Wc$#bOb zs=*=B{N_ZZS{hg%678(IZA+v{Tr%BlNU0(F+4aQ3ed|u@JbGy<@(B@*`3&0WEUqE7 zIx@+`A*jAP)DE%l8&~Y&*0nLBRhxeQeBcSY9f#$9B3q%Fpp^ZG7hky{A{p+y4&8{hR))=v?3S?hVvHOzKJp?|eZt6ZXVa z-a+{9w!3N3a&zcY)SXw)eE4_QGyr6RubqHQF>>L!gtktkZ535dkV6l<=o7rAi#v*5 zP92Yk>xUM#bj+KY2ef`?*tGJ^V7Xt->0d)Rk&OuXw1zC(aRi20qo@(|qWk}ynpC!x>1Pc?+?7OY?XGZkreZMd?u5?F6MC}xkpF0ENba?S?P;g z^7bT`s^+>T2Ww8lB?%TVEiZPqaK(dEs&r4#_!XJ%d6Wp*@Wl;O3R4vK0w1~Rz%6Bc zv-y;h3vfTe1#^)?1Z1N7IC0%s#tiX9xDb@cbRyyDlB3OVpG1o-ypc<1@4)=|bI8Fc z{tgNI=#IGhz_DyNS)_!s$aCkSN-p^Kwoh>p2@~T{2=JWww;mlT$jdsIS7}ZIbo6z-ya}zMUUgepHDy*%=w>9E!oS2%=t~6 z=K>PNG2J)md{yu1mIT#G^?Wf0RR?m&51{3$fw93TOh+2X(m%(4+{Qt4G#!#g16A;q zP8v3pEdOJ%lOSx#4R~^u=%r|p2SlJF=21P_+;?@yoUNK-|hkE{kq`QLxEK(rpH`0 z@UI#F^)qQ1spRi;2g??thp&E8^SV_Z=_^${%oeFx{RBBZF?z`1XDciB3FcKN>mj*m$2mjdN~$am*HN|ab2DhQM*;iAHsC;}J!WPOyv&FNwur!RlepJkuXrYMsJFUl(r1Uchc#d_F!h- zdV43aYGmk4aOiy@YPeql`$v=u%{%F<;J$IRwvxkS;<*@$`HbJemrim~#m(sp@&2=X z9+|tkj{gt+bV0Pv-3tU((7X`L2FHKi)! zb;E_R9o~u*WM2(b%JZHYQrS^O7gdpQRVAUnqdYh>UjSo*z8%=92By^vh_d23gFY(e zWSqVD<;%H~34yci+tJgEwoK0zD09E{^o84lZzP4rp9|~xPf9D$L9(P@Ci3D|Yc5HA z7zFjQiaZ(w^yePH!QlEyxq?2`X0HF?q?+PD0SB47JUfLT_w@`4Cwh?OOu6di@+y#L zi~@DkkhNb1NrL2$7C|o4nLF;U!#7~Bo&TV72RXRex8X%QKRKc-uq3$N`xe%Ly8kYD zk6fY#_k38W&DVYbRb=@2gPQJeLN6sUgna~kJ1olF(M6V|`ei~sBmpIqf@(<@XEOJ!(X1!3e?p) zd0E(~B+`oHdW4qqu0C&{qLY~XzGmc6137v%*9_yo?G{r+?a;J^!g-W+5&y_I>>o14 ze1_}pr>Sbh0NPF_;WJOx4<<;@bUN^ul1CWTEFp98`tX3ds3n5OIhnQFa@tx83E2e zRD3S<#Xbl+li=-foujC6UF;v!I|#rX<9pa}l?Ihr&)h_wp@}>rrqHp_H?u{H+-r6p$&pN2^Rpm(e!b+eLvA$OlYJ#sC(KY8-LLU#Z;ERCdXjf+(v*pYB zKK1%n5p-F7y@7*7q?niRiuHrIeU$gAP@U$)xS8gIWhW;@6WK-ekZJcfX8M$KEHOU( zaOa`Za|*5e2Ln_8_PQIgq9Y8qi_Wa54iW!iuZlOaYw59#vkEzZ(S*d42fL%tOIBTH z=X8hfmNr#uv!-^l!EEXuVb0V$ z`(m!4Y9a$m|Mq^K!;$svhY?_|S4w$gw!UjqwuTQjPN;yKMQvb4A;#E%G*)gEyXVA*$#O-MRroGkMHoy4YjJ~5wE3vgsW(K)^KJ6xC# zK5U7Cxs4yw)|Yex5^TFQVs)$D3o@GJ_fjBh*|d;{hKhAqUtgba9T=NfIXP3ox>t=P z6;RY&96FfZ@v5>ay|TTK|Ec|Y?yl|WdAvOSXoO*$bL>a;(P3xXKoHxv27TDeA~1&_ zHwa+z_nXLDMaCAz3WfJMkwo+4yVO)vGP=6DF$R#RfENskq3vV#Gp#YeyCX1Ppqxx8 zXVtzYb;(LE1T!!mJ67tlsHkm4cN+nU=QF}BS3Y#|LkAYrf;4CQK!<^x`%z*W^g=-c zSC6%3$Xvn5^4Ng5j~orzJ@xq5p>=b>X#HoS;Q}_k2u4TdbZ($jRaNs~jX;JVacM03 zZ(h*pE_jj`fX9~wWH+Rl!w@r4egKAnDNqh3stFi3WUvZRIVAeD**IG~3qKywYUWc* z{=*ZJ)c+?p0JQ|aDq6@M5nJAiZBe5@jjZ4z`~d9{AepIGcnASWK$iFbf8GV$wen)&3@w68kiNXfR)YEbjLN>&}Ck-j-? zsA&oSjk{gr?s}L(Gj+&Bm=l7T6B~|^3{}~G)bRh1-mB8>?b*P2AcxT81Oq^WJG-R> zWcmZSk{g<@V`Q58LMAIC{b6oi-4X*#x*lUgk$dchy84nxyS+{6k=8#54@U4{S_f}w z)tX`+}e}| zPG)GfF-d%-^F+HD{|QL771D27Xq}9d4|y_%O+QuZ_wp>6%QP3jYk=5ps0O{4RUoo( z@(81Z=M%+}%_-oXzdKJw^>TRHxS^GdEriawChqXClaLiRVz3DYm}!6TMCnqI*yJ>+9wI zn}>ZY+!0-N_h~MZKELd{MbKtzJ4G zfq=854$KIblE@Z(C-~@nTW0)Caz7x>M$2KTS1u>t3qBr~6|8}91EX)$6 zzNt5w43r$s(GVHMeb^n*CIgcEGrf09Bn!I6si|a`gK$< ziU(DO#iSIA-JfxyMzrE&T?=a4Zl3TqE98kGt(~}O8Tu9`?ADw#WDj$~^`SU*FxuvK zIDDc$A!!(Hrz!cnpId5Y@I|dk-JdC-Ms#EDd@lDaP=Gg|YlH{c)=|CukppC?>F@db zL}%mcpUoz=K>O&)oS6e17%Rh~1_$qwp&)09x!{Ic&50wCeQJsfJO&o3x}Z+er$9&K zN<^Ce)w;%v-Djm;Cn0SBx00X=shu-D{IauxbfjH_QVLj-?RIi+Fx>jQRZM%|in;uY zR$VI2>Yjq1ZOLGpxe;>P%yoBEG)SDFN1~9r0ML4*L+CsO#cy9y@wwTa0`Cz`pYCOs zkH3QwOV64A{iv%yKB#B^Z`=t@Sa(!&Yio)cr$S3H2rJRxa*BjfV+?c?Ax%;eMcIc~ zyxS)G#6Fjq#RP-o&|?LRBokkBkyBZdbHIQCQyME98_pP%>nEYQaSA&06jfBF+fp0} zD=zE1@9XM6?<`h+Nu=pC8;b1?HuyJN9m%_5C|#HF*NE2x`DbagG7gNpIIH0L|Mcn8 z#ONPK6nfd?oWGjnhZiC&E(jWOih>;xthrpg(LghTTg zRH2QGi~!a5l8G}i@=TQktSM56A6{yY$37H^T&NAw5Y2O80e0)JqF{M;_rmI(dL8OW9DWJTeej6-f%pq@~q>&F|x^$c4 zEjrQ7E~Sm7{tF=ulF*X?b!m~r?Fr72;oBJ^Rr0jLS#M-GRtTy@EO>8evYqurP@-^V z8x{WXV>K93QHVSL#B2(?<}BrnF)*8&tvgcQ)cgg|k68ARJ~#k+q>goPaDd;2Sl4#J z75f|ZMkCmYO2FA(0u6}$g;rXDWjTdiAKg%BjycBKv@XIWc|lIEqS1ec^Cw*;!EN7a zpKf!!F{q5sf=w6a(v>SUy}W(ooE1A)5a%bfoIV3%ZQ0r%T5v8S9j|;YVE?RaLrOm> z@-Yvf>!x@yyEGj96W|l5iKt8$=uuR~ac2#viA&ic&|fROy3}B{H;d>W{=Bo>C>QYZ za+$Q!9qsj2B0g}W{Ry>&8C3wobV&BoqCv3xodqRtoxanQ4{^AGDM4_DE~@2CgJFjK zjZ2r}TRncgiB*DdKik9bWBbsDmWNO1-NQG(5;M4L<{1ocf>KP_;RHKS59}Qd!t`+- z>!))nkA%U6ivkmlpf(7kazCZP*9Rqw6LtbkR}C#zE{h1Q!6I&GaMwT=cT*Ur$=bwk zNcs5og#RxxBIgm7#a`83v5jlp%*e9+JH9)2@Al=<#u@Jnb^_$kL)m4WEL9m zxZJgnfe2ANTc&$s#4YqHlS5 zbbWy3S;!&H%y9x7Akf)q@;iV6X<&<@U{~dW(pZRiuOZLJ@Noam`6+;fY3K;NmAE}g zS|N3b-jKr9>IFW%cg3&#OH-Sl{h$uKS0aT(tyyH_Kw%7VmVjLlIRY@hOi(leYZ%mT zDwy?5z*mG6{mKBR5eGLI7|sU%t^rpq3Q!2&eztWprV@&<@UD;C+=RjYHUlgDXW0o0 zexL8ZM-42n95~N^s?cQz^Ai!-HjKIa+3^20#iH&P`2}>X?N%2Ej#y&!|;mXB4xKG?9ZHCe`t9tJ$#q ziN1t0!e8P8pyzL+nI9%T6+e)HX?y@)g2C zvZ^uns=!l+lriW;UE0;~Q=ljb-B3+!Hy71$g!{azf=ZlR&c|7b*ZgUI#&A5Lb|hwh zDj;E}RO93gCc;p+U^j$_A3=3~@8}SgxzNd8_;qkGz4kq5JD+DK;NJY){jD#mt*COw zZY|!$TM$W3=adi5ql&Bsdy=q8v#IlC=-jK6zT%3QKC+eQ_7FAiTlJHAM~%h0iu86* z5%SofyUT>?SsA}@v$mAcRtAIK_z4HbW4DQ$gi6D8y!24?`R?tLX!L51@1TNE@&}QU zTtjGZA)~FqBe$F0Q38ZM+dU@h51rDzX|Ev*dj+`A+W|43jEub!{+k9K7}FGDx!WpF zS0-X^iHvcuAj5ONhRzM)nioa4XT5CccYXOcF^r^&F5bjhUPys+xYMjA3tnwp0q%4< zNVpY1D3 z6x~M_z~$sVhd!AikQt`(>>rE_aepO1d25!1yt^A1r*$$(H(+SzMz4^W%#!$^NOiix z6w}YCFLA8wVtV97mJku-2R`zsB7)q;krGw3pk!+IYU5_DKOs-9NNTj}!|k0?S!5>t zfJI;6o)^)huL^m~8|AY0lMi*2f5ZhE(#}8FsRIe^!Ws#BFxq-(C0p@^>$6d6*b5hY z{3QDCsDAAJH?jc<i(?cA6mQD$AavyL( z_Wi6f*U$NtiyA3)C7(fw4dy+FIGS-=XU{kszJX9p9Vme`TziE_MwwPOuUe|id8rE2 zaPgl8M4(k{&8XTA(=G}eGCM zy|?0f>^(dkc0xps2d8xIg$2rU_dh5Gw#=`5Dm8R?Pc#uCQumjgIXpVgJmuIzi~!4Y z3Xy3L!T)lQ^jE#;R4Q8qYrKpB{s%Q(lqLe-=eF`Mk}*DbnjAwrvKP_JLH!2VgKuC; zlH8;To_3j~eRKMxqdkq8*^x5L{dipJ$meNruRbUD`DCSFeKfMuaA2mt$Y*|4up~s4 z+BY5y(Hy@e5Y57wI|C5U4^q(SuAL?(zm!7d}kx)5D#Hb;skOW4+9isd2{oEzg)>P--RykoRT`rkf``YcT7`QCW?Z*crn`d>6=PV5O;ryrd;TDsTt7=)3;hwDIhymIk|yMW;FW5myPPO$+rM zf5p0#=JP2x)!j9*IP6w+xbe{a0O$Tb*j5p^FIg@e42OOx_$84sXuY_^#q|MXlhkS~ zwA0iTKc6fH>V98Vb{v@MgdljqSlx+564Ia~``B#plhjv*nHuzW4_A!PhCj)Kx(0u5 zMMW$qmXRAn1L*c=VGPI)qL)b}^l>VUfmDRc?C-TvZs^v6E-dv*FHhgLiy}_o8nIl% ztAG}Dq@xyoQ)4ix#j1fWUM5x1_NQ`V1|9qwQE3Pb#jlCdKRm+xFV94UXke_Lb)n~ENe&J`xl>t8aGa$|>s=-91XWnP(fKm6wi2{+~z8LF(n@&son-p0mB}7SruUKbX&6m~s z`rw5ArvCk|uo>5u{+@@H1QBQ`YZFQ!Im?K=4-@w?yJqf!p`~qeq5dHb!YX z^&6Eziz*SVz&pqb?N`U{ATD29_*sG7m-;5p{#u;oo=;12dSdt;0Oj%Q8c(Kgf!~?= zi}Xu_Vo8~c7wMY;ho_hM=S7c)y+E9)+T}b@o}~-I$v8vH8&GmwyIf}9dH+e`%Y!bY z86?3Bj!!hS5kh4P>5T`=!GGhZqFQ-}A25CrzWeqIjC$Ou&vI!)aE9TZ`L zb+(J4)DK+ZU%>Ye>o61xPK~8}XBGA8M}bG%J`-X{bXwx!p*3;LvvdBYk!hMDFS|Hw z@+5L_R9?H}zx7&+jjas2_}h>|IBUX`G~TD`%_pDo5^pmug&rGivG$)CPVvYiA2@>x-{Y zxZc~&tn-vQgS%N zun|Q3weQ*NJd^uDAM|#RB|JTkDy7B*U&8w=Fl7Dt(3JwUky$YJe;RuWs4UyATlAr& zLAtwJB?Scp1QZ0NK}rRrOF%$CNht{dr6dLE?rx<8q@)``QbGauy8Zrt?7h#}`<(L* zbqsxZ;=XdNx#pVlmL~bYPg0fp9wEPaV7-KdW)fPo-!<6J2pdCrtgN zrBja&DA-v(PA2z1O16yN`rs;Zf&YRF**wEr#?Ot2%J5TJJ+83TI-`EI>uaSq&IP08z0FJS#K8Wj9HO)9fg~lzOoD)9msrgmTc_$_1vlcRcIm>vs~Ab z_FL;?l;^pbv&o{o8c=ukzl37gO)J9}Tx5r_6j4f{bn#={4UkPyN_XG-+}j8RJP}?| zdc~KA==e9cg<3YOJ?a~ZTIz8el?zeSqp~Ohk&Wk@p)XMl&_f_2xneh=ij1NYjAB}D zqOq(W{#~9px9}n(Vybd9cK=S~vnCXXsVX zW$iWGA|c2~9XwLFAN0J-N#pNq%hpP+>S0$eeW6y7V>AFgSpN5 zN*sLs7im#!9Cl3TeAZ*A!~I{Vsm;M(Ui+ah1F$wvf9XkzH`n9lKa@wR8-F3F!{P|U zA)!sngq6p_dHz&Je84v2UVqeH(4je$sF?7I%Idib4oAnY>ln7f=^^0BNO&TK@0kyq_N`x35^&RK8!KZ6!f?ELn$0NK9C`%DJ02bvKLJ=k`vp44#k zQ503LR`rv+RZKl-)X^YDk8Pie8Jpt z6$yeHwos^imm1BD2kY>@R(Z3djg-M6&4rpU4vD20S}LQgwq4TVXk4Wmnxy6fPgLW; zHd%j;djy9ht6zaRpYFK}Ls&KA)5$JmdqqG!eLxvd)2v-onw`$FCzT(25Dp2&V}%^; z63LmjVpqZwjWs?mu-*x4^m&3&@8=;}i8ExQk@3FR;zN5!)9^&H{i^pNzVE4bLLWZB zK*O8tpqNJKaD>_N@f((Fy>=+C9eq!}sf&e(zM`6;!hMTprlqw**35#V0ZN%9WQ(7k zP7GPkXQDb7K%#vU*CvL3zEhnr!xiu8PsiXi^^1 zfYdC(h3NbDG{SL3)ARLT(pic*6;vcb`3xwi-z0tA3k%1Akl?)FH(}9uQ%GBz?mJdT zuOa@fE53})H9TVDiC`!!OCNp~%A(S%*_cr?8WIY+EyN&etu;hbiPZ5Wq7k_`%EzOm zv}w1r=}e-cb&dO>rcAEw+4ey$hh^3_!_t++fN(i}@&qcH^^x%l&f?M^2CsJ{lT*%L zr=;4%{eZPnB{j&G?;Q1TYa@EpIxVg3o7KI{>B1Q8$1*Fb7Wp%QhFB5kFN1zhuRUKw z8Eh9>^s7i0n-JVQGh?kzjSW@LEHQVuL3O^wwJJ2jRv(NEljdcVV%Y8x`s%g&F&yPX zSBZrnt?K={SPRv-wO?Uy-6?-0isYKLZCeRqxc%kCLRY@UK(1g zv46RR-lUC#+K!322MU9%4+Z(gCn^xqdNPyF`i-7{8H&CULHBYE(UmmcdcMI~ue2i9 z(9X5C&`$=UHH=eB8P32H`43j~f^oius%v?^MK;5=;jpUfpx-4!u_Fbzy1nq6TN^Ka z(CPaAsU?X`J^0-zT%^m+*%-9X%f)xJI=-QHSa29vBKx9oLDxYj&7ILSZbnf_ZGMZ1 zNxr#;1f9@;UyKTvr9Y(IdlxR7On=Ew-jcXKweBT7iD!yZ)Qoh%9dAi($;pH(6W=mo zdHX$E2%cD(c=fAZwW7sPEBsl6mvy{V3|`=gDtd*dP6QAUI{E z3>lFiLyTHo&2m6JX_Aglsp)Qr?%WpF0Mbusw zsA;^tym~CQ2iHx_y{`|D+I})3y2E;CKsY>0QOiYu!Bq6#Yg~Xew>5oa>K_HFBZ{2S zub0qb={f!{FF(tTZsZY>XaydgIt`FDv9#f#11!;=2B+}wF9XV zcGixL#j90j6yY#QiG^=w-Vpfy%^02^QNpxtgCxh55|!~`Yx02&3PoU0L&*3w%&A5z zT#8V$HOO-fhvr$pJLxcuZw$O`8}~(U)>U$!z8T;BxqyN#HW8&s;1L`mBez%Z(+rdP znwHk})0LjR6iGlCph-3XRs^gId(w#JH!X>ddMj3wigZdT@y%*t!X2>=c(R; z9(!CjrjlnY6ieVZcV$F&!rnKGKEC23LQO`2Pi-4IxMCY21oqx1#|pF&ZId#r>vUAk zXTQxpDtVDKE{t6Fp1ye_Yql1OlO~t6wUuAv9TF1yb3KkwjleWDvBx^47(+&{J=NP* zU3}rCCns^ZO$lbWuFWCpo7+stW6C+v_=~;tw+0Mnd$9~ge^&?Td2?;!5QfRHB&g;J za7t^u(adSxxX)AJ@k-C4|I66Nql>YR+<5Kv!LjkSoISbO?st_l*F1w?wmyjD;3!*| zdj9mY87567`)y1|H|xiZQSSf=cw;r_;yb0=GPu!MqpciyX&$mqE<#_&2AmSn-YNps zYX>V^PJJm~iDQl1gTl=}HdG|~^3Uxg%^|wV$sn0LYz}c~od0Km{Et%$`~5guv&zgi zp)YYQil`IUlO9b(y&GD3^D7KrG4p9nj&M)zQfT_?@EwMkbxDXoxrWj@7xoX$iBt(z zl8rT+7Du>706H}GEDXBNdHDF$pX}X)IBk7ze=i@A@ZKqteg#kdEWHvenv8#&|8x~C zUdJV?Bh>KMAwePr?c72p2lX=pZ1#W{l2snT7o4!Nmpm0oEybK6>Vu0k#d$9@!JVR{ zf5SeOBp&(^>%}9ME9ruw$r)vV2pN2H=JVSG-Ewp2n<7T8CFV&y^$KoaubPLYU_4LV z9WJD4Z|>Lhx-s`Ylt{$cr+H*fY9MVSZJbxah?lY+-$nMvU#2^nT>Q}o*3UG|e~n_;+TJj> zqNYCeK6Mzgd)s=D#h}p9bIv{*m3Deyp({y%vt3 zqdTB>=Hz5SmMyvZC6Fvd&9M-*P7H9{o4Xs`1x`HPH6lrW?fO`&8xnob-wq)C zwaPKItCUL6QrT$!YFO+F6BIxd)hw2j_PdMmJs-fJ|JwDKk6oYx`uC_i`U*4tF250k`>EgE<%&56pCKa&5G=WXpwt9l zjIG&uTmp&T^|9z=+M5a+FV}o>H6WxS3}4E4=|s<0XnwG8+3j&c*^)9%ePoQhslK(zg$AaUP6PV4&&M2*%!gQT5=6h7w9&Y5(t-5fxNl z)YT(*s%*vuK^1r0*vpQty1@lcq|#?2;fS|mc|-4boW5lweT*A;Cv6AgAJaAGed)?i z+q;{U8RP&fTK@mSir83LL&CxiW)Ph+Q`0m|kf^dAM|X9NVt&@Y%WmA{Lk-c~qTTJ< zopw9m*Xmlxg8(h5aLM21?et_wl7K#%pFhQ<=d9cU5} zOX_!TZfIyYji5}4m&<{J&(yuHbRszbi}q|Li?fzr04-C!eT%rN)YVcD;N;6x*5oPS;H!vRE0`{$xS=|3wOzZxf> zzqd8$E!Rv9W#3za>!$kGps$oc1CTGvP0+ryfJQjzx!Z)d1`;lmvQK^{BqYQ_i^fzx z8G}ZY=IzqXVS63)4|gFym-R;&vfpnEpY8s64Hv>hf??~=WM=?I%O%SLFizua`?xKj z$NZpq@{VgrbcifQD2b)qoM4k?5 zI8{HD`q#qYp=3w#(7*cRnU@i+(R6E~>^PfL2&5O^p7VEPSt_VvvfXF(?k)=}JS!3Z zK;d<8da6MR#BV%aTQ1{x-{Q}*nrXg9+?MbKtg3}Q6|kadDMmK~Tk{zNzcKQ6EI3~k zZoTVF#jglOg`BTE=xP5qfa%7D3$&j-f@yC)-*%Wz%g)HZBv#d@I{%`UVyjDLy3l4aZqCr~3|lTx@i z-~^aGg%yPk-EqMnE_Vf_)<7;7DX#@|nwx>n5UTH;h<`_i!s;Lhzdi>+A2H~4V!Lvs zEWHh9oss?$zAb|a!L$v={5a% zqD=%C8H?CxM?2MV`{SV!md1Ann-3CMt4)5zxJ_j7My$S1l}m_X{jiGsT|J~?K?3AhnJnhL&ouf5v5(aa2N$H_+FRM~X z{ZjF{dj46kZFy^$Du({LakQ9=?RGFb{$7aEnoXNk=K zR0}l0{x?2Ly<`T1Q{8v(iqlK;pGqxIh7%@zJ`{OfyzETXG!3Ao*5X()-vy+#se6 z1D2_Vo>}**b1XHZtY&t~R2+F z{?s$Kl&}KU3zm1>xi- zcr;eq7Lp`+gEE@TqSNUg6gpEA@!IGuCH4ynSablE@>EIW=Gr$@??d4Hn9@M+sLRD) z6@iwP*X-Xf=ciheGt|#$mBfFA@xp-`1^rtFu-a}{wnuiZvM-Y3Gid$tI2qxF1nRBB zFYCLxuS;ZQfKvoLn7=zkcYN~|Km!OvkJ#r;R!zJZJl7jMxd1$%X0S6APn z6kpI)JOP+j(AMz3-2d*-;Arjn>g;*U!!HxYnmM+H_dS@ASj3;|$YQd{=wfHB-S!Lr z-WZc@9+1{vFQBmpVv$hTG7sglX%us`q;*=m?=8)GL;X)f!))oV&%h%5w%(QMbzl1` za3w05$8vTFo@G1w-?Nlx0<*7C1!;ASKWUpgxiQQhgf;rS0qSjlJWoPmKQ|FI!rQdv zSUc&I6%H#rmlre1HC$bWabDhrQt{e1jCJSyXN-5CQOI^wgnKx=e02&1yh@P`nZ{TM zugh(91>Y;`cy{WLkuB1Ec~O$*WP!S3Z+fA=Vd7@t?kI6I)Br5=gKd^9tv1OkZClAlF zunqD}u_y5*5Ol%47bnIAZFbC=RqBI}0p&>@mZRq_Df#7y3#-W~D=V~6pQEV?PJZgQ zR!4DtlA3chBql=H(}nv0ZbKj(i`{XV!uvPJLx}74R#8BT8-{7J{2Fw^_&_^O$X7C` zbG2>Nb#JmN6SC~Cx%e$J7Yp1$#A*q(#sL`LR-FpET*ZR%!)8|azS2J)y#l~m0QRfQ zYJSzYnTYn4qUxT^3JWU<{vlbQLqJvgXjN+$^3}w|)Y6J9;W>3yW4J3od9}bv4 zH0nfprQ&z-nOq;*y!MbJWYv6<~x(JUgW;Ah>m-OM;E%dp^_ve z(bT$&V>3y8Xm#j zODkVl_6Uk3Q#Gy9c>*svf5}6Ih$jtLNrHLww-Lsn2LA>X z4D-{Ifl}B>QTmTp0EAhoH*rZipMuCf6&lJj;xIjJ(8NgcRQcV+EIiy6sG~@}_ys%= z+PWmNp>pY)ww!IvXXMelAh!79A5(S{$P^wg1}L?<(fmZGQW zyDAZdmKczxQ@3YAd1V@-a93BGZx;iI3UeTN|9)o+=2a+!K$ELyp?rfRN>@!zerGXd zml@-4?Gtj}Kc3<{g*W@~dymTjcv>rBSj0bt@gG)(N_U|`zUC!i51$V8@0SOrmqyt( zm)o1TLdBCV6>X${eM%0OXYFIhGn&Ag60;b3nh&dShE`oqgG$=bHwPL=tlq{3H9y#( z*?zur`%Yf2+coLCL{Ww#f2g3k(=v26ey~qjFPL}ucK!1a;a}{GF>XR10f3V38HJlmyYwML=4Yn~h+3|v^lPeI`1FaP0uy;~hx-r!C<8szZlqAnt)ZkK^;|pzlZ2>bk zn5N*;*h$sD0KIVX#A)seN>Q5A%QJTqd0~4hM4?r0zQ>=S-r03Hqaph1iA7zZ5bU8t zYMG$-;^1!szGH(-^%&7$#=9w>t9f)x-`&JO3JA!*E}!}GPt6?u)5td!gIovE;&(@j zuspIl$Rrl~K3W9k!D#x&_6gMK+F609Y}3Nr)d{wf$DW;-XpXsQWv$&HtuKI*oD|;M zP{lho`Z1ol0vBfL!dznQOY9Wv%%VL-5w-`cC!4b_$d-Lx8Jk39zgp{X{5vQT&Mvlp zXV-qvwL66dev_T4vLh2s^%o?zU(>ng9Y1M}8B}gPM13^?YNY~+R74K@T_45sPG-1X z;p~!%_*yXK8?pDZ%-p5!#j%Zp!`e2fofR6fN89>s_i(e-cRx}5CwnB#US1f1aaEKy znXqPoktAP{kp@KYLbn_TLs17IEibrTnySA2#Nw}HEJmB4C6I~`3eTn*Wx~Fp19M7d zZ5y3#UkrTvkhG=rDno@C;h#x36Ut7uyA|i3`LOrCI0TVe4VC@WAf;-q2=y(JUZ<0y z0%$=Mq=*W8u48UJgj27B%nG)!Wrbgc=(TIaL%J9xaX``a{6>IbNughQ{l`G|GLJd;UZ;HJ{AMbsMf3111m=bx&1=Nn2FGz@+RTtUH~I3#1NJ7jP_zesBrgH z@eiiAOQU0J_oBqRMA@`*BM-y05R zZwElGiPAyl%VObens+2$90q?5K&X)AlSb0ElZB%JqGrR~nBPW=ra!Drma_3{%<;Zl zz584j-K$#pp#LwvRm!Pv<*(O-2qz0Ump8A~eD5~7nNH#PSH08Fru56tuIKVR?;Z?W zkwaMD3oB-!yuRCcO#d~tD=k3D1K&HZ)0`jT8K2oRiCHdH62o$>{Oo%x(CUdCHUpfV z;fabw+oi|En?>Hw>a|t#H-DmiPpqH~Y;s~bjKD{eZp?<%lwT}oN6!#Cd?1Dd=#pYg zz0E(7>J_xdTk2IiUO0b7-I$H(_Z5Ha1Y`uhoAk@|GtV15W`;QJFaRITTQ!?;lr)WtU#hW4D^x*G?Mm;KeDWdeUd1E6;`pal=5+6v z?D=+(8C+-+NrtTy_}@zAyZv)!df{Cy|E)~^8qnNe&WsPSIO41nQydQGELih*B0s8Ruw-`sXknexmd$|Uk_+0E)0)udRp_Z$!kRzvNJziV=Ut^K=!1F_eKpOS}r5)KR0r0Co12BlgDTdEt_Q2N>L7&sHn zBfPrZ!#IC5$C~f|$=bWB>-Y75nakuL*`|9v(y3SS6L1xC94e~-+r>o=mMgKwy~}b= z>(^8pU$t-dzg~IT_t5o%iJrI}O7A(n5$e;wAAI~}5aA$t7}p*%;an4NdgmDFg2VTb z?m!j+P>??ogaW=mS~&X=N<%=jKp@8I)<0D86%5Enp!pVL*%3Lc?^%b@`fbN)=5+o; zq5xztw7a<+OEW8z{AP!~eShY;bh)k21{UQ-JxkbM`yN(ljX~IN@1-PCT|5;lRjI$f z)9i!vwZM(5z5aH^$*(|?3&g;0;n7znVws_5x+pI!x@Y9qx3#qXaG_pV7R@VSEJ-w* z5ZDk?l-lYN&0Qn{WHfL)>CG;vG9p=}o5JPs2xU&m#nFSiP(%iz1aRSGi{AzP;E#tK z(@QspN>BL?o=vQ=28+Tp(qf#;1}bO}2dWhB84@-fvBKXb%pEENnR9OCj0vaTy9SBd z!8qS{q>i1GQtREgM2%#1zuV=ItZwZv)ckkPV}QiMG{<+>60~g>^qz{=J{jwHk3nm+ z(I}qt{n#P+4oCik>YMd>9WP&KxjqdcK+MAUA|ki^F&RlmkHsxJud2 z0CP_iuD^Nfl-u%ZW}}E|K>hLP(^}YY2}CN)^%GxiCzYRi9FRKq=0EpR!*(eCYzk)? zluPW>&D}3s4PB;r{8&L%0-#0}JmRzob4$n$yj=KlDc0 zvIzjp)9lvr;|&e}1LIt}Vhy>Pc~axgwdqLn%6d>)DZ0V8QPCdXTev@R;eF%#yZ+F| z(;5R0_h^#L$nv`2tB{67^N8KQeVTR{{MR#;?W7b`x`IUqvA$+DT6 zS;xge?%EZd-`uiIzZ4J^xZSIPDi$8E9bj$hKWG8MJv}#q1J#EYE2{~ zESV2SR4CJ-r|T39;VQ+gxg+MIHo{h-ua^xq|DGqW>kRYM>O+A~J?#u9OTv zG>eSW9;@}u%G#{JFtrm&=LEs8cK5%K5x|X(j1QdQ9RZ0GNe|?p(GRPUXM4~FNiQ5~77r{$AUB6sm)C#Gh@PS>>Q%^;h2S2x#3I191b6Jc z6-!4}08ci(~7j+1c6uOc|jm2GsILKsC(RbAtnV8K!~?DIo`7;biC(m0Nbjd&I<3 zTK2=LJ$+s8DTQtd#n3#7mwi8a1b`}V4+2aj0_n;DtuK20h+1?fk!-^x87~fJfHQY< z45Uz^IQXR#MsP{qXZ80Ud_W$UI@}7$t`ekg_&6O`xQF`+oId5|sX|X=yP^)v6&QQ918NUuU#uXl$q8cm+jK$!R@L%q zo}mIn@D0`SN^#Qic&|skMT@#jqkK0PS3^_#1n55iAHX7W4J+`HfhFe?$c0LU;ulB( z0%(=h=jpjnF7kY3ESEO`d)+%fEJ8{EZc5uh!4)gDnP!MLetE6~ldk;0iUx)n2&+xn zi=i1qY*G{A1V3DB^B{uGUJ4ZFlHDi!iyY$WYJ_}rCJC$`g?H>Tal3zg(@G)u+^ zwo{9MzG7(Yriy#bisqX@_+F#TiZcK)J?Nw?v`D`@sQULHP6|kujR%Ye;=l>YhD2|q z_mvwD=AN%HfDRfWYUkOwMy0Q+s%j{v4=PGf0$X(~jf2VSLwf>+(hvKR0FgmIJ*1+5 z>I<(&2J?i?bpCv_@|av7P>M4=IzjQ&vD#K8;4?v)YgPFZh0FXWlVdICUYUij(|^{+ zxWwLJoKnDH;z}E6Eb?n@6!IR(xE%pv0PsEHNEL{U2y~J)uCeV1c;getjQ5}&>;m=T zWI=I5q6CD)!FRL_$M1cI=uo9S%pKF%NC@;vV&PZnJCe*bN^4fJ(SVWd)usO1J^W=+ zhB^^rKg|Jn1W0f3y~ELEGV^8Fa0o{QNEEK8b@B3}z#2`Pv9|;`8}v@b-S1`&BdlZa zKAD?$L8RZHMW!85QUu*fIbdZX z?^}V$vHEL6aanib@FvL18F#$ijHeRNeHOR&;k_#lHForRRepV z4h(K7)AKH1P;&}aQ8mzas9)#GgDC7mn^)I@<1QGP{ihwU-{N3KNXWs%=M71Q%LW6T z{PRlx7qxM~BUFeQ9=SCh1@jK;0iA&McgMFKm8&?LuP;g6DEOYE6TO-Y9K$_PGyxWV z6nA>bS7RZ5^oe9aI95sJa%MjkWlYIEsU(X)7a=Uz|4B*HGq&2HrC%Xw4o18Hpl44ln8OVX9qkAGMMiKio(84#qj@TQ^q_1#1c?s!yTUS$ zUeSFgRSX%(>-DLwpRot`K*Tc#9HkJw{>RH(?PU@)-Y+609 zv?$o8Q0){3{h-lY=A#vD4O=HD?>EiXhcIK{nx&d;K~zOE!&WwIlS8GGk(VmJoKudk z1}N#StPF<^PHx~#Sf$Za_tKX+Vi5TiirPJGl zH!MLp{%u^`sr}^Sq`)y0(zX*p3iAg|prq`*xS9TYfAF|VN6#N3W&5NNwGfrtZ*sf5 zO5q~XAm`4jcwk)KpX}}-cBMyyD}*D^%M+s!|5}6CWAf8bmYx1jrZedov6S97Zk$b@ z#auyws?1#)`rFWT5q#&#c{;1kwvef1e@24YOdGB+y+gD6x>V-Ss}=U^dGx%ywld@8 zFHeJW79aWnWI8#&1k@M+%8(v1pHwe2ra8$VE+M8Kr2h&SGY~#zkgUyDXz8ey!%~ZejdaPBv-ud+BxX~4S0ljVYqh-&Jfms}ZDXzyLM(_jQ z{Jp5xPtRaqAR%;feEU8qkl(ejf7T=wr7H)zE3{bH96=eH^N!JRaeww-1jg97)NAk4 z>Z88WLzjB&@9rz?pTjdetSO{scuUi}Jn@wQ4(WIwQKLOEqFmSlS2=Hf%6bLfOk&N~ zI%s@-X+_RmCBa=OcE;o(RH$V3SQQgWtGnY*{ZDp}L~MOrFEh=E1YwrHhQ1};=B7w- zU52)qw(Gfoufm92Lb{D-J%0#*POl8or}EH!Ca}|duMjU)(&><(3;^vF2@%v2R28C9mqYv>7;{yA95A+s zrl>ajML3)~af^!aF>_CD=Z4Pb{1Q#;%5u?s%w2L_#u)OhjwTU!uIDSM>HTD0Q=$BB z^nw)r>LXdFuhi0qx#1>s8TQkY$8bf8Odpl5c@$Ms?P|zzXp7b-kGhq7Tr2WYFwJZq z;d=DdMx|hb=hqQ?Y7w-*dAt0ry-PzL+@5{KVZ+#>sXltJnqehj>%PgoLXs54ocHU( zV1|T;hv%_X+TGLN|73kA9U5Kws)gD4`0yiEPlV#3PY6V143S=;AOadBY5`^Bn{XGE zG|#>w$?zA(*_4DmsqIDuntD($*_dt;*d}+`YXCT9=qUwE*j9hxE>=YkHoAei9tv&( zNh7QD3G42aAOL~S7o`to7W+6so2M7bXm7tA-aQH@LOSk&(Y)tKV{N@x=Mmki1l%9L z-a;{h070J`eNAYeJpTEjym#0G^EV<$Dl3wbKgCmr%{V+=ea@rP~VqP0-2*DDB+VkNO#2lbycwvZ>c{ zIc9eUmFIS!KlotcHF06;gSAuCzbxy+0bvg$A znmVl%h0{L*XuIJ(;!z}vcj59qXrJ|-JBQ1dmjC6VbG~=ouS@C)p&6h&ZcaPHG(o3C zysR7Zz~JFpDaesr%eO4ZhL#CaFSqv&spiR(%5gJl`f>hn0u4wgL&X?mY_e)BoCaC~ zh7zBLa_#);<1O2V!VU)6Ejln%H{ND0A8JeV4@7uR1?KrXr4`GWQ~`yj zn}bJbUsoS*MADo55mF1|s(5J-5i$9qAh}`Ry6Y%Z`U!6Ok}R zO~n37zdrfX5i>z0A99j)>`)1J1t{({KTdG2=~0C&Oi8rNFRK=MeW4p79NIZdhaBy9 z);&Axfws|fE;_EOemk?Jk}ZZVrFSCgEYuqQ?G3OW^n7zkf`6QR&?;0yV6OI86bIk}g4bl%C$;S>T`M~G_!RzDG^eRQPlqd5ZT4k$-=^tC5xo`# zlqQlY6wSR@eVaK@siddam*tZ`0aOGPOQ;atD3r8G37px0B7=dq6P_$gJha0>O)g`E z8^qbZR!4k*x+AP7$o~S~xdO2qis4YE6szkE>n*&$?I@NNbb|5tdV{wnR3_8Y>9@auaNj=V&r*fe2=B=K7 zWiv?xI%rrqq*hZUr`-M$NJu!Xp8z9Z2%;q=h33gYO`{8Cup7jpp*S4e`g4WWSX@Tq(%UQw0PD zbiXD%Yu?&TasXL?yHw0by(?`jD&BS#RBP8oY5Co_pqdBH9E)!GkOnQgTe@4W;|5lt z0hg$0>8lur(b}^@9x0e{v!GUH}OZFN-?w)u3wLJG|V2I{*)%i zf?^JwPEL)i8;sBQ*DAU8j}5`bDB6J;XyAXfw4d1M@IW~mas_)oRVF5k-?pFE47#NP;|Px_Q{lvK+FANrcESxr8#2#0Wsr*i63u3L<* z8Cy>-2kVe5(2~GTVg#sE5U;|C%!*F<%=5VEVuc!rmblPoK#r9^qX@k2kI9emZh5i! zHec6dKpH1ASV9*k@dn_qbFnBDIwMrM^2hT_!!N2kw38}X5HMG^mwVQhd{x)XE*wr@ zj0M)l231au$@je>@Y022PV5OKl9NhBU;{M{`eCIC<guZ*WJdv zQs71XLG^uhn*^<`x= z^;rLjd#OzQ+Z&~={dnIN4}ld5CCn#)76C3V-?#+i1MC#b8-}08jD-R~UZMMa1hhT$ z@RnWSpByW=B9xcsoZHei``sC?Y??%Roi(Rw!zaF}mNj)C5Y5@hW&4EO0C2+hV*I4L z{LH_hE#Jt##!oeLbO>s5G!C{3^gak5sQy|3)bN|oW6{EPR zP=Io(m2si9Nx->J9w;Ki3PgCGQV9)m4eLlgJYac$!}T>zH8W;_*@i=Hor3HY)Zuuf zqdZ(tkTQpei;g?sKX)x?fLS2(8vNY;0rTL2X)EazuPm<9UqLRWAE;$mWlP2 z@%P3McwiXm;Dj)QuoQpxIttLRP)!rEqsigVc2zb*&%PCVjm#?c^40!|Sh3iIiqEoA z7y?fXE>B6933IB?cTP+M$naZ~v&2(1DDQI`O6?HqQ{ivD#q6cJGv}am^<;LbxAwSC zZ!XzoP*S!^My|=NOa&)NC5*F}c#W>{U9xpoZP1U#0Pz_1r$Z0bn)*EIaX*YzEE+XZ zEp6kjZ|=2^eGBaEXRz8iYERQPYeS{mzPgI`WUV;8jzv#APF3Eeo;X-gbSZHhC&Hl3 zcP+9t^VlM-v7U;8DrtCf>2|U~pwd0@0def(ksBT?HS?!8iA-9N-@a-UOQkqn3#onS z54mEfQLjTnP$-?H7BV^ctF8iF66@Ejvp8tXMyoQ~*Sc41wyCeT9KB#kEn-YAJf8Qy zRo}4GHSyNt?!9tZrko)Cxu8e7QyR}dVM#^cNkvGVq^7FMSFxSd%{j8CeWoKwRVp<2 z$#J)O_V>le-vM|Xef`(a_(;aHozl7_PVc*Pk7yiIT3qu<;nCCJ^8z`A45VJ&nF#ns zv+)ZjxN?W~kz9j1S=IJ;9Il6`yBaF3Pdvj7L~ls1w0UtEO}N*g(R^0IZnwtuJpA-}1K2v}4umRa2Dhq3cpbZv>w}it{B^v8T~V(d1%uZky{0 zuD?#4s%FE=4#t-s%6EP2n5$?ss2d9zv^E^tz)#zE#`iN7TF?+|DHzF(wkK?IpvLo6 z>5`Bj`$e7d*iXi6UM6>p7k6o!Gr49{sk4*v)Wf2Tlm}OezHao@RY77-ss17BBAlLh zH>RTJFW}o7cMSfJF6pw_yQx@PGcg)Der(bb`ll>t>tbCBTb7lTHQ*lR(;r(eD;+Ex z*UxWyH6L$vWxsP=w59_YFFP9F<=SRtTxVuy|W&E6f;6#x!AJ6BhH39 z(}-r&^o=FB2p5CkN1oQ@vTa-mw`{^jAGAtdiO1JREwT(riXXHW5MwtR&KL_$r}{lG zsUaotn@nq!d9NeU=Xr_6v8XldTcoLj)n9kad^&$Gu{SDLSo^Iz)}Aq5^QdD9kc}5$ zOrlyPI6c=S!C)pO)nu~qrZ=*iv9jL@5;YEE^p%}$%>1i>|00?8{pUT2xrxXn-6_-2 zs?4g|Ta2sPDzYC1Y6MfW9ju-Ur>oF4&7wA^>Y^+dyqo7ug$(J?(R-95|EhZZo#lOz zEM6J$r#BO#M5tVTJ@fWc-uaXprv{bw|#5??+y9 zsm-m%^5t^=rD9}VAH6FbKi*e4d1FgwZ73BjrF@TtgupWPem^o@pEG|pF!0l_zWkAD zFS=NhP6(fqY_n_nC0AkentNKekm*;OdEp<-Dwl0>4Qi(9hUYiTQQ^YK%x2t3f6d&# z-6~5(d$B$Dhss>UI{N9=Me93UUkjAN?}!lU{nU9nn&(?jEQGpyr`&Xk&<1CAgI%R< zS?em1u~pZG6fvj7WsMmU`U{4H%R)kUcK3#kv0CgOyWqYHF-MmFW-K|=)-&tvbvb5c zZ12Ct&67*E?K;)bf>$LOxtbi?r^mv+^a@yQCY_0Vb32#m((ofvYe?^@eJ&^tvly;A z)Ge2<$-?ZnO7ubcJ=>G0$zsQlCkxl2n}=D0y+$53%Mq>Yfjp z(Ml2cfd|65-XBl2e7c5p!b|Jdc-d)BZcJ%V+l1Y2_~|)c??hRyR2buK_O!w5?-N>s z|9ZiaKicwsrK!-^cEkAXh2O{MT|Y)0iG-gn{}vX;FlFCkK*r2Ro=ooS<@_Ll-z~n; zAC^})mOnF{Pm;MxU}3i?Kl=C3T-Qk11JE3;RALUZuX_8plI*(u${1NA_PQMvL=@}? zr`vB_HB2;Agldcz=2H(VQC{|6g=jsKEz#{LWd#LS{~?8_$FXk8$5YvcWQouE&E|W| zkTR--Pk+=U)$CPV`H$ce|YE;-()98eaDt8e4kPliFPx zDT9oKxW=a}Im=J?mp7J1h3Q_s(AvTZC-}Fw>y{?|%wpIcUO_3L|NCa^k+JV>15|Qa zx$&zP!Z82)hx$Dya3**Hg0TPnP>4AJvdv(IUgx0@pJTSGSIImZ89{VTUQw|#?a3TC znkvVT9}WMQ9J>o&U|7f+9TCJsG~pfrqg?_l<3Fk;1%X>cMtZv0(40t8e{Oz$?9Bur z6UHuDGI(zuHh2l5Qc+-3dl>}Xx|cu4?tyt5cv_i*OosnOB)baulkaygI1)UHfH4hu zOG<$@w04l5mxmV^7`QJIry2-78h{xR{`N91rUJ_cJpme~(Px|YBT%m3o$%*+fp(V8*A z!I%-v2?CAd(H}k_`jV!g6JPW9;!i1b$bhcPJjnDer8Pe=G71Mi|7mP|yjZv>h>x+- z(rS{T?8N9stwlF~{AiCHA0KbuaGm`s3gD+OXtHVP>Jl5Qe#%LeH#AH-`&3xiSm^n0 zKv;Ou)4&7O-d+&uG0Tr$%Az#A3V z%|0@qi4qFgV7L^GMloA8nFO$hNp=J9P&hw_SBJ)$R9RV7A*0}mKMa&{4! z!6?fSPn1m1ry!|3ZfKZH<{y0aCXIV zwRl1P!=k6CJNwDoZZQqUY1!wt7%QcOtU@|o7xLy;k1oR(?)-OHvSv`XqddSA$`x@I zb(Ko_s{LC3-w*%im3~3^TrZL5>!dzH{d*bpzs-ZZ3^u6(`J;cQKC*Sv+7O}t#}*dy z#p8(ic5ZI2MxCl=W=#$C@pl*E%a<>kR!<`)UQ5s?V)mr2uC4?mL%}n){qdEB>vN|> z$h!G5Iy!0!swKj-5hc^x;6Q`n>E#uCpp(%CmPl~ggG(b9)6&741`y0!L{vnaov(nh z9~hHp=B#yH3b2{UC&0SUbDr_ z?R0~ous_(PP!nJyo?D1*`UGQzEyn!-=1#C3d;q>yrO$roIPEX1l)~hjmFg+Hc+vP( zI}8uhWGKK_4D6YcNp45dlg|DAR;}RaSqJa!{2ttxUPGl?^ZtESUOv7~Fp=d0qcwpK zJKNh7px8+P8+oyO@CF$OKNlKbl73m*3)$MTEd`HgCosD2MT;7| zxu5E>i0cHBiiDmwNG@H<8Q}rBSFBS#8=HJsn~TN1=fLU>fw99yErMvO^Lvn0yr&$) zUS!-zTl?kqYY?CfW4hQ4HtPmyQg)Cc3?PA7glman@!(WLTN^nuPv5C#%fq4o3oh~k z{X=UT8w?b9bD}_1kiyf`vu(;0M5vYT-AkBGD<~iUHETx+dgu3Wn73`aEx|e6uJI9@rj9{AhjCIkaX?oWov6|l#y?c$&q`{hAUj-EL+|V z4nm1ZNt4*srtLw5$Z+g|Eg=&ukdszoAHNS5hdJP7jt#@HB8=iy$RA^c+W}^EzZ9-m z@OK4+Tt2<>b(Vk@(EZ&758y|@&FJ~^1@HQYq@=K#nwkZI-9Oc`Ic$g(Y zztTDmHVhRQM3aHX7VFKMCh)MmYqxr%5MS9v_`-`|fk(x|!($Y*Y_oY2U+>b5zsmW! zxg@j^O0bUfldrq0Z%vfm0y_f4P@uLC&N*lrB&~Pfq3n)iAOq?TwIgl}TxTKBtL6kk zxX~#o{T#~n!(X%Sbi$5Vd1C6eJ;edGU?-=8zJdv6bWxY7Q4~jC?+CpH)wJ|6>rS|C zI>Fv9uIT71N=(rNAJ(y(2#va&(*SLkg?6MJQ6u ztg*2%Gq`e2`W<}y;pS>v+B1cl2%+V`Y6sD@QoCs}6evC`-n&N({s@R;Bh);ZXd|#; zn2YzoRdE5_FX3=QAscD^mnSU0MVNWST>u=rOW_*ha7YMQpM$(CBUe{fH+Xi|nBSn{ z5`@JlKX5ui`9UDy?k=9Jo@3+#%M{kT2go3^s9Z*w!a{%t;ua7~iR+6pL7 zhl2MpJ3BjQnzzC)>&lG$e0-om5Z4zCTfOxi7#s{9|DgymoWf3U$huo8PE<%nszG7b zY@^rSM$U)DcH)~INY5-t`qHkftYE|+dp1IKv@??7y1i5aJpab#Wl}|Y$Wb~mRbr4b zGOTPIkXQt}cWlIc9ZvC+^NW%)#5o-N-PI@8`q2gxlcP*77Ngbfwg)ujY#*SPQBzn5^^qK1RU|Kmr_ z!^6Y0+}ulINrbR3TR?3Yi5-wBfncCJj!$>_$8YZ&P~%6>QBP?DDZ!GbBb@&lW&e^6 zGrWPMjr1lEB{bSgdBu)%o888=$#>R{d#zHrKud_+-wqTrq z_xElG**(+Y48>g!X@uYgo2$(e6CV#RDkUYt!RH`p{pZgvkbBHJa{wE>)|F`Q7 Z{=R!4>-4Q-Mg#@_sVJ%|6v>;u`XBD<9}NHi literal 0 HcmV?d00001