add log
This commit is contained in:
root
@@ -29,22 +29,17 @@ command:
|
||||
- --reasoning-parser
|
||||
- qwen3
|
||||
- --enable-prefix-caching
|
||||
# multi_system 兼容: 合并多/乱序 system 消息到首位, 消除 "System message must
|
||||
# be at the beginning" 4xx (数据集 34 条多 system 请求, 原模板 raise 致 15 条 4xx)。
|
||||
# 模型目录只读, patch_ops.sh 把改后模板部署到 /workspace/ (见 Dockerfile WORKDIR)。
|
||||
- --chat-template
|
||||
- /workspace/chat_template_multi_system.jinja
|
||||
env:
|
||||
- name: VLLM_ENGINE_ITERATION_TIMEOUT_S
|
||||
value: 3600
|
||||
# corex runtime paths (must match patch_ops.sh)
|
||||
- name: PYTHONPATH
|
||||
value: /usr/local/corex/lib64/python3/dist-packages
|
||||
- name: LD_LIBRARY_PATH
|
||||
value: /usr/local/corex/lib64:/usr/local/iluvatar/lib64:/usr/local/openmpi/lib
|
||||
- name: PATH
|
||||
value: /usr/local/corex/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/local/openmpi/bin
|
||||
# five GatedDeltaNet/MoE/attention optimizations (default ON; =0 to disable)
|
||||
- name: MOE_NATIVE
|
||||
value: '1'
|
||||
- name: CHUNK_PARALLEL
|
||||
|
||||
@@ -42,6 +42,21 @@ from vllm.model_executor.models.interfaces import HasInnerState, SupportsLoRA
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Profiling debug prints (locate where startup hangs after weight load).
|
||||
# Gate on PROF_TRACE=1 (default on, since we are diagnosing a startup hang).
|
||||
# Each print flushes; grep "[PROF]" in the server log.
|
||||
# ---------------------------------------------------------------------------
|
||||
import os as _os_prof
|
||||
import sys as _sys_prof
|
||||
import time as _time_prof
|
||||
_PROF_TRACE = _os_prof.environ.get("PROF_TRACE", "1") != "0"
|
||||
_t0 = _time_prof.time()
|
||||
def _prof(msg):
|
||||
if _PROF_TRACE:
|
||||
print(f"[PROF] {_time_prof.time()-_t0:7.2f}s {msg}", file=_sys_prof.stderr, flush=True)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Pure-PyTorch DeltaNet kernels (fallbacks from transformers 5.2.0)
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -125,8 +140,10 @@ def _torch_chunk_gated_delta_rule(
|
||||
if _os.environ.get("LOOP1_NATIVE", "1") != "0":
|
||||
try:
|
||||
_eye = torch.eye(chunk_size, dtype=attn.dtype, device=attn.device)
|
||||
_prof(f" chunk-gdn loop1 solve_triangular start shape={tuple(attn.shape)}")
|
||||
attn = torch.linalg.solve_triangular(
|
||||
-attn, _eye, upper=False, unitriangular=True)
|
||||
_prof(f" chunk-gdn loop1 solve_triangular done")
|
||||
_loop1_done = True
|
||||
except Exception as _e:
|
||||
import sys as _sys, traceback as _tb
|
||||
@@ -173,12 +190,14 @@ def _torch_chunk_gated_delta_rule(
|
||||
B = Ktil.transpose(-1, -2) @ value # [b,h,NC,k,v]
|
||||
M = Qg - attn_i_all @ P_i # [b,h,NC,cs,k]
|
||||
N = attn_i_all @ value # [b,h,NC,cs,v]
|
||||
_prof(f" chunk-gdn loop2 parallel batched done NC={NC}")
|
||||
S = last_state
|
||||
S_starts = torch.empty(batch, num_heads, NC, k_dim, v_dim,
|
||||
device=query.device, dtype=query.dtype)
|
||||
for i in range(NC):
|
||||
S_starts[:, :, i] = S
|
||||
S = A[:, :, i] @ S + B[:, :, i]
|
||||
_prof(f" chunk-gdn loop2 serial scan done ({NC} iters)")
|
||||
last_state = S
|
||||
core_out = M @ S_starts + N # [b,h,NC,cs,v]
|
||||
_done = True
|
||||
@@ -430,6 +449,7 @@ class GatedDeltaNet(nn.Module):
|
||||
mixed_qkv_conv = F.silu(mixed_qkv_conv)
|
||||
# (1, seq_len, local_conv_dim)
|
||||
mixed_qkv_conv = mixed_qkv_conv.squeeze(0).transpose(0, 1).unsqueeze(0)
|
||||
_prof(f" L{self.layer_idx} GDN conv1d done seq_len={seq_len}")
|
||||
|
||||
q, k, v = torch.split(
|
||||
mixed_qkv_conv,
|
||||
@@ -455,6 +475,7 @@ class GatedDeltaNet(nn.Module):
|
||||
_DNN_CHUNK = 4096
|
||||
cur_state = temporal_state[si:si + 1].clone()
|
||||
core_out_parts = []
|
||||
_prof(f" L{self.layer_idx} GDN chunk-scan start (DNN_CHUNK={_DNN_CHUNK}, nchunks={(seq_len + _DNN_CHUNK - 1)//_DNN_CHUNK})")
|
||||
for sc_start in range(0, seq_len, _DNN_CHUNK):
|
||||
sc_end = min(sc_start + _DNN_CHUNK, seq_len)
|
||||
c_out, cur_state = _torch_chunk_gated_delta_rule(
|
||||
@@ -468,6 +489,7 @@ class GatedDeltaNet(nn.Module):
|
||||
use_qk_l2norm_in_kernel=True,
|
||||
)
|
||||
core_out_parts.append(c_out)
|
||||
_prof(f" L{self.layer_idx} GDN chunk-scan done")
|
||||
if cur_state is not None:
|
||||
temporal_state[si].copy_(cur_state[0])
|
||||
# [1, seq_len, num_v_heads, head_v_dim]
|
||||
@@ -717,7 +739,9 @@ class Qwen3_5FullAttention(nn.Module):
|
||||
# rope: q=(T, local_num_heads*head_dim), k=(T, 1*head_dim) — mirrors 27B
|
||||
q, k = self.rotary_emb(positions, q, k)
|
||||
|
||||
_prof(f" L{self.layer_idx} ATTN flash start tokens={q.shape[0]}")
|
||||
attn_out = self.attn(q, k, v, kv_cache, attn_metadata)
|
||||
_prof(f" L{self.layer_idx} ATTN flash done")
|
||||
|
||||
# Multiply by sigmoid gate before output projection
|
||||
attn_out = attn_out * torch.sigmoid(gate.float()).to(attn_out.dtype)
|
||||
@@ -938,6 +962,7 @@ class Qwen3_5MoeSparseBlock(nn.Module):
|
||||
def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
|
||||
router_logits, _ = self.gate(hidden_states)
|
||||
routed_out = self._pure_pytorch_experts(hidden_states, router_logits)
|
||||
_prof(f" MoE routed-experts done tokens={hidden_states.shape[0]}")
|
||||
|
||||
gate_up, _ = self.shared_expert_gate_up(hidden_states)
|
||||
shared_out = self.act_fn(gate_up)
|
||||
@@ -948,7 +973,9 @@ class Qwen3_5MoeSparseBlock(nn.Module):
|
||||
|
||||
out = routed_out + shared_out
|
||||
if self.experts.tp_size > 1:
|
||||
_prof(f" MoE all_reduce start tp_size={self.experts.tp_size}")
|
||||
out = tensor_model_parallel_all_reduce(out)
|
||||
_prof(f" MoE all_reduce done")
|
||||
return out
|
||||
|
||||
|
||||
@@ -1013,16 +1040,22 @@ class Qwen3_5DecoderLayer(nn.Module):
|
||||
hidden_states, residual = self.input_layernorm(hidden_states, residual)
|
||||
|
||||
if self.layer_type == "linear_attention":
|
||||
_prof(f"L{self.layer_idx} GDN-start tokens={hidden_states.shape[0]}")
|
||||
hidden_states = self.linear_attn(
|
||||
hidden_states, attn_metadata, conv_state, temporal_state)
|
||||
_prof(f"L{self.layer_idx} GDN-done")
|
||||
else:
|
||||
_prof(f"L{self.layer_idx} ATTN-start tokens={hidden_states.shape[0]}")
|
||||
hidden_states = self.self_attn(
|
||||
positions, hidden_states, kv_cache, attn_metadata)
|
||||
_prof(f"L{self.layer_idx} ATTN-done")
|
||||
|
||||
hidden_states, residual = self.post_attention_layernorm(
|
||||
hidden_states, residual)
|
||||
|
||||
_prof(f"L{self.layer_idx} MLP-start")
|
||||
hidden_states = self.mlp(hidden_states)
|
||||
_prof(f"L{self.layer_idx} MLP-done")
|
||||
|
||||
return hidden_states, residual
|
||||
|
||||
@@ -1185,6 +1218,7 @@ class Qwen3_5ForCausalLM(nn.Module, HasInnerState, SupportsLoRA):
|
||||
intermediate_tensors: Optional[IntermediateTensors] = None,
|
||||
**kwargs,
|
||||
) -> torch.Tensor:
|
||||
_prof(f"ForCausalLM.forward START input_ids={tuple(input_ids.shape)} prefill={getattr(attn_metadata,'num_prefill_tokens',0)}")
|
||||
if self.mamba_cache is None:
|
||||
if self.scheduler_config is not None:
|
||||
max_batch_size = _get_graph_batch_size(
|
||||
@@ -1267,6 +1301,7 @@ class Qwen3_5ForCausalLM(nn.Module, HasInnerState, SupportsLoRA):
|
||||
self._gdn_prefix_cache.popitem(last=False)
|
||||
# ── End save ────────────────────────────────────────────────────────────
|
||||
|
||||
_prof(f"ForCausalLM.forward END hidden={tuple(hidden_states.shape)}")
|
||||
return hidden_states
|
||||
|
||||
def compute_logits(
|
||||
|
||||
Reference in New Issue
Block a user