diff --git a/computility-run.yaml b/computility-run.yaml index d04963d..16569f3 100644 --- a/computility-run.yaml +++ b/computility-run.yaml @@ -29,22 +29,17 @@ command: - --reasoning-parser - qwen3 - --enable-prefix-caching - # multi_system 兼容: 合并多/乱序 system 消息到首位, 消除 "System message must - # be at the beginning" 4xx (数据集 34 条多 system 请求, 原模板 raise 致 15 条 4xx)。 - # 模型目录只读, patch_ops.sh 把改后模板部署到 /workspace/ (见 Dockerfile WORKDIR)。 - --chat-template - /workspace/chat_template_multi_system.jinja env: - name: VLLM_ENGINE_ITERATION_TIMEOUT_S value: 3600 - # corex runtime paths (must match patch_ops.sh) - name: PYTHONPATH value: /usr/local/corex/lib64/python3/dist-packages - name: LD_LIBRARY_PATH value: /usr/local/corex/lib64:/usr/local/iluvatar/lib64:/usr/local/openmpi/lib - name: PATH value: /usr/local/corex/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/local/openmpi/bin - # five GatedDeltaNet/MoE/attention optimizations (default ON; =0 to disable) - name: MOE_NATIVE value: '1' - name: CHUNK_PARALLEL diff --git a/qwen3_6_scripts/qwen3_5.py b/qwen3_6_scripts/qwen3_5.py index 5bf7289..e717ada 100644 --- a/qwen3_6_scripts/qwen3_5.py +++ b/qwen3_6_scripts/qwen3_5.py @@ -42,6 +42,21 @@ from vllm.model_executor.models.interfaces import HasInnerState, SupportsLoRA logger = init_logger(__name__) +# --------------------------------------------------------------------------- +# Profiling debug prints (locate where startup hangs after weight load). +# Gate on PROF_TRACE=1 (default on, since we are diagnosing a startup hang). +# Each print flushes; grep "[PROF]" in the server log. +# --------------------------------------------------------------------------- +import os as _os_prof +import sys as _sys_prof +import time as _time_prof +_PROF_TRACE = _os_prof.environ.get("PROF_TRACE", "1") != "0" +_t0 = _time_prof.time() +def _prof(msg): + if _PROF_TRACE: + print(f"[PROF] {_time_prof.time()-_t0:7.2f}s {msg}", file=_sys_prof.stderr, flush=True) + + # --------------------------------------------------------------------------- # Pure-PyTorch DeltaNet kernels (fallbacks from transformers 5.2.0) # --------------------------------------------------------------------------- @@ -125,8 +140,10 @@ def _torch_chunk_gated_delta_rule( if _os.environ.get("LOOP1_NATIVE", "1") != "0": try: _eye = torch.eye(chunk_size, dtype=attn.dtype, device=attn.device) + _prof(f" chunk-gdn loop1 solve_triangular start shape={tuple(attn.shape)}") attn = torch.linalg.solve_triangular( -attn, _eye, upper=False, unitriangular=True) + _prof(f" chunk-gdn loop1 solve_triangular done") _loop1_done = True except Exception as _e: import sys as _sys, traceback as _tb @@ -173,12 +190,14 @@ def _torch_chunk_gated_delta_rule( B = Ktil.transpose(-1, -2) @ value # [b,h,NC,k,v] M = Qg - attn_i_all @ P_i # [b,h,NC,cs,k] N = attn_i_all @ value # [b,h,NC,cs,v] + _prof(f" chunk-gdn loop2 parallel batched done NC={NC}") S = last_state S_starts = torch.empty(batch, num_heads, NC, k_dim, v_dim, device=query.device, dtype=query.dtype) for i in range(NC): S_starts[:, :, i] = S S = A[:, :, i] @ S + B[:, :, i] + _prof(f" chunk-gdn loop2 serial scan done ({NC} iters)") last_state = S core_out = M @ S_starts + N # [b,h,NC,cs,v] _done = True @@ -430,6 +449,7 @@ class GatedDeltaNet(nn.Module): mixed_qkv_conv = F.silu(mixed_qkv_conv) # (1, seq_len, local_conv_dim) mixed_qkv_conv = mixed_qkv_conv.squeeze(0).transpose(0, 1).unsqueeze(0) + _prof(f" L{self.layer_idx} GDN conv1d done seq_len={seq_len}") q, k, v = torch.split( mixed_qkv_conv, @@ -455,6 +475,7 @@ class GatedDeltaNet(nn.Module): _DNN_CHUNK = 4096 cur_state = temporal_state[si:si + 1].clone() core_out_parts = [] + _prof(f" L{self.layer_idx} GDN chunk-scan start (DNN_CHUNK={_DNN_CHUNK}, nchunks={(seq_len + _DNN_CHUNK - 1)//_DNN_CHUNK})") for sc_start in range(0, seq_len, _DNN_CHUNK): sc_end = min(sc_start + _DNN_CHUNK, seq_len) c_out, cur_state = _torch_chunk_gated_delta_rule( @@ -468,6 +489,7 @@ class GatedDeltaNet(nn.Module): use_qk_l2norm_in_kernel=True, ) core_out_parts.append(c_out) + _prof(f" L{self.layer_idx} GDN chunk-scan done") if cur_state is not None: temporal_state[si].copy_(cur_state[0]) # [1, seq_len, num_v_heads, head_v_dim] @@ -717,7 +739,9 @@ class Qwen3_5FullAttention(nn.Module): # rope: q=(T, local_num_heads*head_dim), k=(T, 1*head_dim) — mirrors 27B q, k = self.rotary_emb(positions, q, k) + _prof(f" L{self.layer_idx} ATTN flash start tokens={q.shape[0]}") attn_out = self.attn(q, k, v, kv_cache, attn_metadata) + _prof(f" L{self.layer_idx} ATTN flash done") # Multiply by sigmoid gate before output projection attn_out = attn_out * torch.sigmoid(gate.float()).to(attn_out.dtype) @@ -938,6 +962,7 @@ class Qwen3_5MoeSparseBlock(nn.Module): def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: router_logits, _ = self.gate(hidden_states) routed_out = self._pure_pytorch_experts(hidden_states, router_logits) + _prof(f" MoE routed-experts done tokens={hidden_states.shape[0]}") gate_up, _ = self.shared_expert_gate_up(hidden_states) shared_out = self.act_fn(gate_up) @@ -948,7 +973,9 @@ class Qwen3_5MoeSparseBlock(nn.Module): out = routed_out + shared_out if self.experts.tp_size > 1: + _prof(f" MoE all_reduce start tp_size={self.experts.tp_size}") out = tensor_model_parallel_all_reduce(out) + _prof(f" MoE all_reduce done") return out @@ -1013,16 +1040,22 @@ class Qwen3_5DecoderLayer(nn.Module): hidden_states, residual = self.input_layernorm(hidden_states, residual) if self.layer_type == "linear_attention": + _prof(f"L{self.layer_idx} GDN-start tokens={hidden_states.shape[0]}") hidden_states = self.linear_attn( hidden_states, attn_metadata, conv_state, temporal_state) + _prof(f"L{self.layer_idx} GDN-done") else: + _prof(f"L{self.layer_idx} ATTN-start tokens={hidden_states.shape[0]}") hidden_states = self.self_attn( positions, hidden_states, kv_cache, attn_metadata) + _prof(f"L{self.layer_idx} ATTN-done") hidden_states, residual = self.post_attention_layernorm( hidden_states, residual) + _prof(f"L{self.layer_idx} MLP-start") hidden_states = self.mlp(hidden_states) + _prof(f"L{self.layer_idx} MLP-done") return hidden_states, residual @@ -1185,6 +1218,7 @@ class Qwen3_5ForCausalLM(nn.Module, HasInnerState, SupportsLoRA): intermediate_tensors: Optional[IntermediateTensors] = None, **kwargs, ) -> torch.Tensor: + _prof(f"ForCausalLM.forward START input_ids={tuple(input_ids.shape)} prefill={getattr(attn_metadata,'num_prefill_tokens',0)}") if self.mamba_cache is None: if self.scheduler_config is not None: max_batch_size = _get_graph_batch_size( @@ -1267,6 +1301,7 @@ class Qwen3_5ForCausalLM(nn.Module, HasInnerState, SupportsLoRA): self._gdn_prefix_cache.popitem(last=False) # ── End save ──────────────────────────────────────────────────────────── + _prof(f"ForCausalLM.forward END hidden={tuple(hidden_states.shape)}") return hidden_states def compute_logits(