diff --git a/qwen3_6_scripts/qwen3_5.py b/qwen3_6_scripts/qwen3_5.py index ca42760..7fa060a 100644 --- a/qwen3_6_scripts/qwen3_5.py +++ b/qwen3_6_scripts/qwen3_5.py @@ -3,6 +3,9 @@ # Text-only (no VL, no MTP). from collections import OrderedDict +from contextlib import contextmanager +import os +import time from typing import Dict, Iterable, List, Optional, Tuple import torch @@ -41,6 +44,59 @@ from vllm.model_executor.models.interfaces import HasInnerState, SupportsLoRA logger = init_logger(__name__) +_ENGINEX_PROFILE_ENABLED = os.getenv("ENGINEX_PROFILE_DECODE", "0") == "1" +_ENGINEX_PROFILE_EVERY = int(os.getenv("ENGINEX_PROFILE_EVERY", "32")) +_ENGINEX_PROFILE_SYNC = os.getenv("ENGINEX_PROFILE_SYNC", "1") != "0" +_enginex_profile_stats: Dict[str, List[float]] = {} +_enginex_profile_steps = 0 +_enginex_profile_mode = "unknown" + + +def _enginex_profile_active() -> bool: + return _ENGINEX_PROFILE_ENABLED and torch.cuda.is_available() + + +def _enginex_profile_sync() -> None: + if _ENGINEX_PROFILE_SYNC: + torch.cuda.synchronize() + + +@contextmanager +def _enginex_profile(label: str): + if not _enginex_profile_active(): + yield + return + label = f"{_enginex_profile_mode}.{label}" + _enginex_profile_sync() + start = time.perf_counter() + try: + yield + finally: + _enginex_profile_sync() + elapsed_ms = (time.perf_counter() - start) * 1000.0 + stat = _enginex_profile_stats.setdefault(label, [0.0, 0.0]) + stat[0] += elapsed_ms + stat[1] += 1.0 + + +def _enginex_profile_log(mode: str) -> None: + global _enginex_profile_steps + if not _enginex_profile_active(): + return + _enginex_profile_steps += 1 + if _enginex_profile_steps % max(_ENGINEX_PROFILE_EVERY, 1) != 0: + return + tp_rank = get_tensor_model_parallel_rank() + parts = [] + for label, (total_ms, count) in sorted( + _enginex_profile_stats.items(), + key=lambda item: item[1][0], + reverse=True): + avg_ms = total_ms / max(count, 1.0) + parts.append(f"{label}: total={total_ms:.2f}ms avg={avg_ms:.3f}ms n={int(count)}") + logger.info("[ENGINEX_PROFILE_QWEN] rank=%d steps=%d mode=%s %s", + tp_rank, _enginex_profile_steps, mode, " | ".join(parts)) + # --------------------------------------------------------------------------- # Pure-PyTorch DeltaNet kernels (fallbacks from transformers 5.2.0) @@ -613,44 +669,48 @@ class Qwen3_5FullAttention(nn.Module): total_tokens = hidden_states.shape[0] # q_proj output includes gate (dim doubled) - qg, _ = self.q_proj(hidden_states) # (total, local_num_heads * head_dim * 2) - qg = qg.view(total_tokens, self.local_num_heads, self.head_dim * 2) - q = qg[:, :, :self.head_dim].reshape(total_tokens, -1) - gate = qg[:, :, self.head_dim:].reshape(total_tokens, -1) + with _enginex_profile("full_attn.qkv_proj"): + qg, _ = self.q_proj(hidden_states) # (total, local_num_heads * head_dim * 2) + qg = qg.view(total_tokens, self.local_num_heads, self.head_dim * 2) + q = qg[:, :, :self.head_dim].reshape(total_tokens, -1) + gate = qg[:, :, self.head_dim:].reshape(total_tokens, -1) - k, _ = self.k_proj(hidden_states) # (total, proj_kv_heads * head_dim) - v, _ = self.v_proj(hidden_states) + k, _ = self.k_proj(hidden_states) # (total, proj_kv_heads * head_dim) + v, _ = self.v_proj(hidden_states) # q_norm on local Q heads - q = self.q_norm.forward_cuda( - q.view(total_tokens, self.local_num_heads, self.head_dim) - .contiguous()).view(total_tokens, -1) + with _enginex_profile("full_attn.norm_rope"): + q = self.q_norm.forward_cuda( + q.view(total_tokens, self.local_num_heads, self.head_dim) + .contiguous()).view(total_tokens, -1) - # GQA-aware TP: select rank-local KV head BEFORE k_norm and rope so - # that ixformer kernels always see num_kv_heads=1 (same as 27B path). - # Doing k_norm/rope on 2 KV heads (proj_kv_heads=2) triggers ixformer - # paths that can produce NaN; restricting to 1 head avoids the issue. - if self.q_per_kv_global is not None: - tp_rank = get_tensor_model_parallel_rank() - kv_idx = (tp_rank * self.local_num_heads) // self.q_per_kv_global - k = (k.view(total_tokens, self.proj_kv_heads, self.head_dim) - [:, kv_idx, :].contiguous()) # (T, head_dim) — 1 head - v = (v.view(total_tokens, self.proj_kv_heads, self.head_dim) - [:, kv_idx, :].contiguous()) # (T, head_dim) — 1 head + # GQA-aware TP: select rank-local KV head BEFORE k_norm and rope so + # that ixformer kernels always see num_kv_heads=1 (same as 27B path). + # Doing k_norm/rope on 2 KV heads (proj_kv_heads=2) triggers ixformer + # paths that can produce NaN; restricting to 1 head avoids the issue. + if self.q_per_kv_global is not None: + tp_rank = get_tensor_model_parallel_rank() + kv_idx = (tp_rank * self.local_num_heads) // self.q_per_kv_global + k = (k.view(total_tokens, self.proj_kv_heads, self.head_dim) + [:, kv_idx, :].contiguous()) # (T, head_dim) — 1 head + v = (v.view(total_tokens, self.proj_kv_heads, self.head_dim) + [:, kv_idx, :].contiguous()) # (T, head_dim) — 1 head - # k_norm on the (now always 1) rank-local KV head - k = self.k_norm.forward_cuda( - k.view(total_tokens, self.local_num_kv_heads, self.head_dim) - .contiguous()).view(total_tokens, -1) + # k_norm on the (now always 1) rank-local KV head + k = self.k_norm.forward_cuda( + k.view(total_tokens, self.local_num_kv_heads, self.head_dim) + .contiguous()).view(total_tokens, -1) - # rope: q=(T, local_num_heads*head_dim), k=(T, 1*head_dim) — mirrors 27B - q, k = self.rotary_emb(positions, q, k) + # rope: q=(T, local_num_heads*head_dim), k=(T, 1*head_dim) — mirrors 27B + q, k = self.rotary_emb(positions, q, k) - attn_out = self.attn(q, k, v, kv_cache, attn_metadata) + with _enginex_profile("full_attn.paged_attention"): + attn_out = self.attn(q, k, v, kv_cache, attn_metadata) # Multiply by sigmoid gate before output projection - attn_out = attn_out * torch.sigmoid(gate.float()).to(attn_out.dtype) - output, _ = self.o_proj(attn_out) + with _enginex_profile("full_attn.gate_o_proj"): + attn_out = attn_out * torch.sigmoid(gate.float()).to(attn_out.dtype) + output, _ = self.o_proj(attn_out) return output @@ -754,12 +814,13 @@ class Qwen3_5MoeSparseBlock(nn.Module): Output is partial (pre-all-reduce), same contract as FusedMoE with reduce_results=False. """ - # Routing: softmax → topk → renormalise - routing_weights = torch.softmax(router_logits.float(), dim=-1) - topk_weights, topk_ids = torch.topk( - routing_weights, self.top_k, dim=-1) # (T, top_k) - topk_weights = topk_weights / topk_weights.sum(dim=-1, keepdim=True) - topk_weights = topk_weights.to(hidden_states.dtype) + # Routing: softmax -> topk -> renormalise + with _enginex_profile("moe.routing_topk"): + routing_weights = torch.softmax(router_logits.float(), dim=-1) + topk_weights, topk_ids = torch.topk( + routing_weights, self.top_k, dim=-1) # (T, top_k) + topk_weights = topk_weights / topk_weights.sum(dim=-1, keepdim=True) + topk_weights = topk_weights.to(hidden_states.dtype) w13 = self.experts.w13_weight # (E, 2*I, H) w2 = self.experts.w2_weight # (E, H, I) @@ -771,59 +832,90 @@ class Qwen3_5MoeSparseBlock(nn.Module): # gate_up: 1 large GEMM (1,H) × (K*2*I,H)^T → (1, K*2*I) # down: 1 bmm (K,H,I) @ (K,I,1) → (K,H) # Total: 3 kernel launches vs previous 16 (top_k*2). - eids = topk_ids[0] # (K,) - ws = topk_weights[0].to(hidden_states.dtype) # (K,) - w13_sel = w13[eids] # (K, 2*I, H) - w2_sel = w2[eids] # (K, H, I) + with _enginex_profile("moe.routed_decode_experts"): + eids = topk_ids[0] # (K,) + ws = topk_weights[0].to(hidden_states.dtype) # (K,) + w13_sel = w13[eids] # (K, 2*I, H) + w2_sel = w2[eids] # (K, H, I) - H = hidden_states.shape[-1] + H = hidden_states.shape[-1] - gate_up = F.linear( - hidden_states, - w13_sel.reshape(-1, H), # (K*2*I, H) — contiguous after indexing - ) # (1, K*2*I) - gate_up = gate_up.view(self.top_k, -1) # (K, 2*I) - gate, up = gate_up.chunk(2, dim=-1) # (K, I) each - act = F.silu(gate) * up # (K, I) + gate_up = F.linear( + hidden_states, + w13_sel.reshape(-1, H), # (K*2*I, H) - contiguous after indexing + ) # (1, K*2*I) + gate_up = gate_up.view(self.top_k, -1) # (K, 2*I) + gate, up = gate_up.chunk(2, dim=-1) # (K, I) each + act = F.silu(gate) * up # (K, I) - # bmm: (K,H,I) @ (K,I,1) → (K,H,1) → (K,H) - expert_out = torch.bmm(w2_sel, act.unsqueeze(-1)).squeeze(-1) # (K, H) + # bmm: (K,H,I) @ (K,I,1) -> (K,H,1) -> (K,H) + expert_out = torch.bmm(w2_sel, act.unsqueeze(-1)).squeeze(-1) # (K, H) - out = (expert_out * ws.unsqueeze(-1)).sum(0, keepdim=True).to( - hidden_states.dtype) # (1, H) + out = (expert_out * ws.unsqueeze(-1)).sum(0, keepdim=True).to( + hidden_states.dtype) # (1, H) + elif T <= 2: + # Fast path: tiny decode batch. With max_num_seqs=2, normal decode + # often has T=2 and should not fall back to the Python per-expert + # loop used for long prefill chunks. + with _enginex_profile("moe.routed_tiny_batch_experts"): + H = hidden_states.shape[-1] + flat_eids = topk_ids.reshape(-1) # (T*K,) + flat_ws = topk_weights.reshape(-1).to(hidden_states.dtype) + + w13_sel = w13[flat_eids] # (T*K, 2*I, H) + w2_sel = w2[flat_eids] # (T*K, H, I) + x = (hidden_states[:, None, :] + .expand(T, self.top_k, H) + .reshape(-1, 1, H)) # (T*K, 1, H) + + gate_up = torch.bmm( + x, w13_sel.transpose(1, 2)).squeeze(1) # (T*K, 2*I) + gate, up = gate_up.chunk(2, dim=-1) + act = F.silu(gate) * up # (T*K, I) + expert_out = torch.bmm( + w2_sel, act.unsqueeze(-1)).squeeze(-1) # (T*K, H) + + out = (expert_out * flat_ws.unsqueeze(-1)).view( + T, self.top_k, H).sum(1).to(hidden_states.dtype) else: # General path (prefill / multi-seq): loop over unique active experts. # At most T*top_k unique experts, always <= num_experts. - out = torch.zeros_like(hidden_states) - unique_eids = topk_ids.view(-1).unique().tolist() - for eid in unique_eids: - eid = int(eid) - mask = (topk_ids == eid) # (T, top_k) - tok_ids, topk_pos = mask.nonzero(as_tuple=True) - tokens = hidden_states[tok_ids] # (n, H) - gate_up = F.linear(tokens, w13[eid]) # (n, 2*I) - gate, up = gate_up.chunk(2, dim=-1) - act = F.silu(gate) * up # (n, I) - expert_out = F.linear(act, w2[eid]) # (n, H) - weights = topk_weights[tok_ids, topk_pos].unsqueeze(-1) - out.index_add_(0, tok_ids, (expert_out * weights).to(out.dtype)) + with _enginex_profile("moe.routed_prefill_experts"): + out = torch.zeros_like(hidden_states) + unique_eids = topk_ids.view(-1).unique().tolist() + for eid in unique_eids: + eid = int(eid) + mask = (topk_ids == eid) # (T, top_k) + tok_ids, topk_pos = mask.nonzero(as_tuple=True) + tokens = hidden_states[tok_ids] # (n, H) + gate_up = F.linear(tokens, w13[eid]) # (n, 2*I) + gate, up = gate_up.chunk(2, dim=-1) + act = F.silu(gate) * up # (n, I) + expert_out = F.linear(act, w2[eid]) # (n, H) + weights = topk_weights[tok_ids, topk_pos].unsqueeze(-1) + out.index_add_(0, tok_ids, (expert_out * weights).to(out.dtype)) return out # partial, all-reduce done in forward() def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: - router_logits, _ = self.gate(hidden_states) - routed_out = self._pure_pytorch_experts(hidden_states, router_logits) + with _enginex_profile("moe.gate"): + router_logits, _ = self.gate(hidden_states) + with _enginex_profile("moe.routed_total"): + routed_out = self._pure_pytorch_experts(hidden_states, router_logits) - gate_up, _ = self.shared_expert_gate_up(hidden_states) - shared_out = self.act_fn(gate_up) - shared_out, _ = self.shared_expert_down(shared_out) - # Scalar sigmoid gate (Qwen2-MoE / Qwen3.5-MoE style) - gate_score, _ = self.shared_expert_gate(hidden_states) # (T, 1) - shared_out = shared_out * torch.sigmoid(gate_score) + with _enginex_profile("moe.shared_expert"): + gate_up, _ = self.shared_expert_gate_up(hidden_states) + shared_out = self.act_fn(gate_up) + shared_out, _ = self.shared_expert_down(shared_out) + # Scalar sigmoid gate (Qwen2-MoE / Qwen3.5-MoE style) + gate_score, _ = self.shared_expert_gate(hidden_states) # (T, 1) + shared_out = shared_out * torch.sigmoid(gate_score) - out = routed_out + shared_out + with _enginex_profile("moe.combine"): + out = routed_out + shared_out if self.experts.tp_size > 1: - out = tensor_model_parallel_all_reduce(out) + with _enginex_profile("moe.tp_all_reduce"): + out = tensor_model_parallel_all_reduce(out) return out @@ -883,21 +975,27 @@ class Qwen3_5DecoderLayer(nn.Module): ) -> Tuple[torch.Tensor, torch.Tensor]: if residual is None: residual = hidden_states - hidden_states = self.input_layernorm(hidden_states) + with _enginex_profile("layer.input_norm"): + hidden_states = self.input_layernorm(hidden_states) else: - hidden_states, residual = self.input_layernorm(hidden_states, residual) + with _enginex_profile("layer.input_norm"): + hidden_states, residual = self.input_layernorm(hidden_states, residual) if self.layer_type == "linear_attention": - hidden_states = self.linear_attn( - hidden_states, attn_metadata, conv_state, temporal_state) + with _enginex_profile("layer.linear_attention"): + hidden_states = self.linear_attn( + hidden_states, attn_metadata, conv_state, temporal_state) else: - hidden_states = self.self_attn( - positions, hidden_states, kv_cache, attn_metadata) + with _enginex_profile("layer.full_attention"): + hidden_states = self.self_attn( + positions, hidden_states, kv_cache, attn_metadata) - hidden_states, residual = self.post_attention_layernorm( - hidden_states, residual) + with _enginex_profile("layer.post_attn_norm"): + hidden_states, residual = self.post_attention_layernorm( + hidden_states, residual) - hidden_states = self.mlp(hidden_states) + with _enginex_profile("layer.mlp"): + hidden_states = self.mlp(hidden_states) return hidden_states, residual @@ -934,6 +1032,9 @@ class Qwen3_5Model(nn.Module): conv_states: torch.Tensor, # (num_linear_layers, batch, ...) temporal_states: torch.Tensor, # (num_linear_layers, batch, ...) ) -> torch.Tensor: + global _enginex_profile_mode + mode = "prefill" if attn_metadata.num_prefill_tokens > 0 else "decode" + _enginex_profile_mode = mode hidden_states = self.embed_tokens(input_ids) residual = None @@ -961,6 +1062,7 @@ class Qwen3_5Model(nn.Module): attn_idx += 1 hidden_states, _ = self.norm(hidden_states, residual) + _enginex_profile_log(mode) return hidden_states diff --git a/vllm/worker/model_runner.py b/vllm/worker/model_runner.py index ee4442f..29839c5 100644 --- a/vllm/worker/model_runner.py +++ b/vllm/worker/model_runner.py @@ -2,9 +2,11 @@ import dataclasses import gc import inspect import itertools +import os import time import warnings import weakref +from contextlib import contextmanager from dataclasses import dataclass from typing import (TYPE_CHECKING, Any, Callable, Dict, List, Optional, Set, Tuple, Type, TypeVar, Union) @@ -62,6 +64,56 @@ if TYPE_CHECKING: logger = init_logger(__name__) +_ENGINEX_PROFILE_ENABLED = os.getenv("ENGINEX_PROFILE_DECODE", "0") == "1" +_ENGINEX_PROFILE_EVERY = int(os.getenv("ENGINEX_PROFILE_EVERY", "32")) +_ENGINEX_PROFILE_SYNC = os.getenv("ENGINEX_PROFILE_SYNC", "1") != "0" +_enginex_profile_stats: Dict[str, List[float]] = {} +_enginex_profile_steps = 0 + + +def _enginex_profile_active() -> bool: + return _ENGINEX_PROFILE_ENABLED and torch.cuda.is_available() + + +def _enginex_profile_sync() -> None: + if _ENGINEX_PROFILE_SYNC: + torch.cuda.synchronize() + + +@contextmanager +def _enginex_profile(label: str): + if not _enginex_profile_active(): + yield + return + _enginex_profile_sync() + start = time.perf_counter() + try: + yield + finally: + _enginex_profile_sync() + elapsed_ms = (time.perf_counter() - start) * 1000.0 + stat = _enginex_profile_stats.setdefault(label, [0.0, 0.0]) + stat[0] += elapsed_ms + stat[1] += 1.0 + + +def _enginex_profile_log(mode: Optional[str]) -> None: + global _enginex_profile_steps + if not _enginex_profile_active(): + return + _enginex_profile_steps += 1 + if _enginex_profile_steps % max(_ENGINEX_PROFILE_EVERY, 1) != 0: + return + parts = [] + for label, (total_ms, count) in sorted( + _enginex_profile_stats.items(), + key=lambda item: item[1][0], + reverse=True): + avg_ms = total_ms / max(count, 1.0) + parts.append(f"{label}: total={total_ms:.2f}ms avg={avg_ms:.3f}ms n={int(count)}") + logger.info("[ENGINEX_PROFILE_MODEL_RUNNER] steps=%d mode=%s %s", + _enginex_profile_steps, mode, " | ".join(parts)) + LORA_WARMUP_RANK = 8 _BATCH_SIZE_ALIGNMENT = 8 # all the token sizes that **can** be captured by cudagraph. @@ -1633,7 +1685,9 @@ class ModelRunner(GPUModelRunnerBase[ModelInputForGPUWithSamplingMetadata]): model_input.prompt_adapter_requests, model_input.prompt_adapter_mapping) - self.attn_state.begin_forward(model_input) + profile_mode = "prompt" if model_input.is_prompt else "decode" + with _enginex_profile(f"{profile_mode}.attn_begin_forward"): + self.attn_state.begin_forward(model_input) # Currently cuda graph is only supported by the decode phase. assert model_input.attn_metadata is not None @@ -1661,16 +1715,17 @@ class ModelRunner(GPUModelRunnerBase[ModelInputForGPUWithSamplingMetadata]): model_forward_end = torch.cuda.Event(enable_timing=True) model_forward_start.record() - with set_forward_context(model_input.attn_metadata): - hidden_or_intermediate_states = model_executable( - input_ids=model_input.input_tokens, - positions=model_input.input_positions, - kv_caches=kv_caches, - attn_metadata=model_input.attn_metadata, - intermediate_tensors=intermediate_tensors, - **MultiModalInputs.as_kwargs(multi_modal_kwargs, - device=self.device), - **seqlen_agnostic_kwargs) + with _enginex_profile(f"{profile_mode}.model_forward"): + with set_forward_context(model_input.attn_metadata): + hidden_or_intermediate_states = model_executable( + input_ids=model_input.input_tokens, + positions=model_input.input_positions, + kv_caches=kv_caches, + attn_metadata=model_input.attn_metadata, + intermediate_tensors=intermediate_tensors, + **MultiModalInputs.as_kwargs(multi_modal_kwargs, + device=self.device), + **seqlen_agnostic_kwargs) if (self.observability_config is not None and self.observability_config.collect_model_forward_time): @@ -1695,20 +1750,23 @@ class ModelRunner(GPUModelRunnerBase[ModelInputForGPUWithSamplingMetadata]): torch.tensor(model_forward_time + orig_model_forward_time)) return hidden_or_intermediate_states - logits = self.model.compute_logits(hidden_or_intermediate_states, - model_input.sampling_metadata) + with _enginex_profile(f"{profile_mode}.compute_logits"): + logits = self.model.compute_logits(hidden_or_intermediate_states, + model_input.sampling_metadata) if not self.is_driver_worker: return [] if model_input.async_callback is not None: - model_input.async_callback() + with _enginex_profile(f"{profile_mode}.async_callback"): + model_input.async_callback() # Sample the next token. - output: SamplerOutput = self.model.sample( - logits=logits, - sampling_metadata=model_input.sampling_metadata, - ) + with _enginex_profile(f"{profile_mode}.sample"): + output: SamplerOutput = self.model.sample( + logits=logits, + sampling_metadata=model_input.sampling_metadata, + ) if (self.observability_config is not None and self.observability_config.collect_model_forward_time and output is not None): @@ -1741,6 +1799,7 @@ class ModelRunner(GPUModelRunnerBase[ModelInputForGPUWithSamplingMetadata]): output.hidden_states = hidden_states + _enginex_profile_log(profile_mode) return [output] diff --git a/worklogs/decode_analysis_report_2026-07-14.md b/worklogs/decode_analysis_report_2026-07-14.md index a12288a..c4a863b 100644 --- a/worklogs/decode_analysis_report_2026-07-14.md +++ b/worklogs/decode_analysis_report_2026-07-14.md @@ -535,3 +535,135 @@ decode microbench 结果： 4. 用 `ENGINEX_PROFILE_DECODE=1` 这类环境变量控制插桩，只在短压测时开启，避免污染正式结果。 初步判断：custom all-reduce 不是第一大瓶颈；更可能的主战场是 xFormers decode attention、MoE 小 batch kernel、以及 TP 下大量小 kernel / 同步造成的低 GPU 利用率。 + +## 2026-07-14：代码级 profiling 与第一轮 MoE 优化 + +### profiling 插桩 + +新增环境变量控制的 profiling： + +- `ENGINEX_PROFILE_DECODE=1`：开启 profiling。 +- `ENGINEX_PROFILE_EVERY=N`：每 N 次 model forward 打印一次累计统计。 +- `ENGINEX_PROFILE_SYNC=1`：每段计时前后 `torch.cuda.synchronize()`，用于定位 GPU 时间。 + +插桩位置： + +- `vllm/worker/model_runner.py` + - `attn_state.begin_forward` + - `model_forward` + - `compute_logits` + - `sample` +- `qwen3_6_scripts/qwen3_5.py` + - `full_attention`: qkv projection / norm+rope / paged attention / gate+o_proj + - `linear_attention`: GatedDeltaNet 整层 + - `MoE`: gate / routing topk / routed experts / shared expert / TP all-reduce + - `DecoderLayer`: norm / attention / MLP + +注意：profiling 强制同步会显著拖慢请求，因此 profiling 结果只用于定位瓶颈，不作为真实性能分数。 + +本地归档： + +- `worklogs/remote_results/2026-07-14-code-profile/server_profile_mode_eager_custom_ar_seq2_b8192.log` +- `worklogs/remote_results/2026-07-14-code-profile/profile_mode_eager_custom_ar_short_c1_t24_r1.json` + +### 关键 profiling 结果 + +短请求 `c1, max_tokens=24`，修正标签后，rank0 在 step=24 的主要累计耗时： + +| 模块 | 总耗时 | 平均单层/次 | 次数 | 判断 | +| --- | ---: | ---: | ---: | --- | +| `prefill.moe.routed_prefill_experts` | 5239.90ms | 65.50ms | 80 | 首 token 慢的最大来源 | +| `prefill.layer.linear_attention` | 2944.76ms | 49.08ms | 60 | prefill 第二大来源 | +| `decode.layer.mlp` | 1456.57ms | 1.66ms | 880 | decode 最大来源 | +| `decode.layer.linear_attention` | 942.63ms | 1.43ms | 660 | decode 第二大来源 | +| `decode.moe.routed_total` | 785.65ms | 0.89ms | 880 | MLP 中 routed expert 为主 | +| `decode.moe.routed_decode_experts` | 540.24ms | 0.61ms | 880 | 单 token MoE expert 计算 | +| `decode.layer.full_attention` | 322.06ms | 1.46ms | 220 | full attention 不是第一瓶颈 | +| `decode.moe.tp_all_reduce` | 265.92ms | 0.30ms | 880 | 通信有成本，但不是最大项 | + +`ModelRunner` 粗粒度： + +```text +decode.model_forward avg ~= 145.6ms +decode.compute_logits avg ~= 1.17ms +decode.sample avg ~= 1.01ms +``` + +结论：decode 慢主要发生在模型 forward 内部；logits 和 sampler 不是主瓶颈。 + +### 为什么 GPU 算力打不满 + +当前路径的 GPU 利用率低，不是因为单个大矩阵乘算不过来，而是因为每 token 被拆成大量小工作： + +1. **MoE 没有真正 fused kernel** + - 注释里已经说明 BI-V100 上缺少 `vllm_moe_topk_softmax / vllm_invoke_fused_moe_kernel`。 + - 当前 routed expert 是纯 PyTorch 实现。 + - prefill 通用路径按 expert 做 Python 循环，长 prompt 下会产生大量小 GEMM 和 CPU/GPU 同步点。 + +2. **decode batch 太小** + - `max_num_seqs=2` 时每步只有 1-2 token。 + - 小 batch 下矩阵乘规模小，kernel launch、Python 调度、TP 同步成本占比很高。 + +3. **模型结构有大量 GatedDeltaNet linear_attention 层** + - profiling 显示 linear_attention 在 prefill 和 decode 都是大头之一。 + - 这部分不是标准 paged attention，不能靠换 xFormers attention backend 直接解决。 + +4. **TP all-reduce 不是第一瓶颈，但放大了小 kernel 问题** + - decode MoE all-reduce 单次约 0.30ms。 + - 单次看不大，但每层一次、每 token 多次累积，且会让 rank 间等待更明显。 + +### 第一轮针对性优化：MoE tiny-batch fast path + +发现：原代码只有 `T == 1` 的 MoE decode fast path。一旦并发 decode `T == 2`，会落入通用 `routed_prefill_experts` 路径： + +```python +unique_eids = topk_ids.view(-1).unique().tolist() +for eid in unique_eids: + ... +``` + +这条路径适合大 prefill，但不适合 `max_num_seqs=2` 的小批量 decode。 + +本轮新增 `T <= 2` fast path： + +- 将 `T * top_k` 个选中 expert 展平。 +- 用两次 batched `torch.bmm` 计算 gate/up 和 down。 +- 避免 Python per-expert loop。 +- 保留 `T == 1` 原 fast path 不变。 + +### 优化结果 + +对比同口径 `short c2, max_tokens=128, requests=4`： + +| 版本 | 成功率 | TTFT P90 | per-request Output TPS P10 | Aggregate Output TPS | +| --- | ---: | ---: | ---: | ---: | +| 优化前 custom all-reduce on | 100% | 1.46s | 4.22 | 8.07 | +| tiny-batch MoE fast path | 100% | 4.42s | 5.95 | 10.60 | + +decode 聚合吞吐提升约 **31%**。TTFT 变差可能来自冷缓存/加载后首次请求抖动，后续需要用多轮 warmup 后再复测。 + +本地归档： + +- `worklogs/remote_results/2026-07-14-code-profile/decode_tiny_batch_moe_short_c2_t128_r4.json` +- `worklogs/remote_results/2026-07-14-code-profile/server_tiny_batch_moe_eager_custom_ar_seq2_b8192.log` + +### 下一步优化方向 + +优先级从高到低： + +1. **MoE prefill 路径** + - 当前 `prefill.moe.routed_prefill_experts` 是 TTFT 最大来源。 + - 需要把 Python per-expert loop 替换成更批量化的 grouped GEMM / batched GEMM。 + - 官方负载长上下文输入占比极高，提升 prefill 会直接改善 TTFT 和 weighted throughput。 + +2. **GatedDeltaNet linear_attention** + - decode 和 prefill 都是大头。 + - 需要进一步拆分 projection、conv/update、state update、out_proj，确认是 recurrent update 还是投影占主。 + +3. **更稳的并发策略** + - tiny-batch MoE 已证明 `T=2` 能受益。 + - 后续可测试 `max_num_seqs=3/4`，但受 100K 上下文 KV cache 与 TTFT 影响，需要小心。 + +4. **TP all-reduce 合并** + - 当前 MoE 每层 routed+shared 后做一次 all-reduce。 + - 如果后续能将若干小通信或 residual 路径合并，可能进一步改善 decode 抖动，但优先级低于 MoE/linear_attention 计算本体。 diff --git a/worklogs/remote_results/2026-07-14-code-profile/decode_tiny_batch_moe_short_c2_t128_r4.json b/worklogs/remote_results/2026-07-14-code-profile/decode_tiny_batch_moe_short_c2_t128_r4.json new file mode 100644 index 0000000..146e0de --- /dev/null +++ b/worklogs/remote_results/2026-07-14-code-profile/decode_tiny_batch_moe_short_c2_t128_r4.json @@ -0,0 +1,75 @@ +{ + "created_at": "2026-07-14T12:30:35", + "label": "tiny_batch_moe_short_c2_t128_r4", + "url": "http://127.0.0.1:1111", + "model": "llm", + "prompt_mode": "short", + "with_tools": false, + "tool_count": 0, + "concurrency": 2, + "requests": 4, + "max_tokens": 128, + "wall_sec": 48.280083537101746, + "success_rate": 1.0, + "ttft_p50_sec": 2.915760966949165, + "ttft_p90_sec": 4.415460329316557, + "output_tps_p10_per_request": 5.954912943833064, + "output_tps_p50_per_request": 6.032240452288079, + "aggregate_output_tps": 10.604786953331262, + "prompt_tokens": 156, + "cached_tokens": 64, + "completion_tokens": 512, + "reasoning_tokens": 512, + "chars": 1796, + "monitor": null, + "results": [ + { + "ok": true, + "elapsed_sec": 25.909583542495966, + "ttft_sec": 4.414824679493904, + "completion_tokens": 128, + "prompt_tokens": 39, + "cached_tokens": 0, + "reasoning_tokens": 128, + "output_tps": 5.95494003053556, + "chars": 449, + "error": null + }, + { + "ok": true, + "elapsed_sec": 25.91063128784299, + "ttft_sec": 4.415732750669122, + "completion_tokens": 128, + "prompt_tokens": 39, + "cached_tokens": 0, + "reasoning_tokens": 128, + "output_tps": 5.954901335246281, + "chars": 449, + "error": null + }, + { + "ok": true, + "elapsed_sec": 22.367535073310137, + "ttft_sec": 1.4166972544044256, + "completion_tokens": 128, + "prompt_tokens": 39, + "cached_tokens": 32, + "reasoning_tokens": 128, + "output_tps": 6.109540874040597, + "chars": 449, + "error": null + }, + { + "ok": true, + "elapsed_sec": 22.36741546355188, + "ttft_sec": 1.4165842793881893, + "completion_tokens": 128, + "prompt_tokens": 39, + "cached_tokens": 32, + "reasoning_tokens": 128, + "output_tps": 6.109542808819567, + "chars": 449, + "error": null + } + ] +} \ No newline at end of file diff --git a/worklogs/remote_results/2026-07-14-code-profile/profile_eager_custom_ar_short_c1_t48_r1.json b/worklogs/remote_results/2026-07-14-code-profile/profile_eager_custom_ar_short_c1_t48_r1.json new file mode 100644 index 0000000..3fce51d --- /dev/null +++ b/worklogs/remote_results/2026-07-14-code-profile/profile_eager_custom_ar_short_c1_t48_r1.json @@ -0,0 +1,39 @@ +{ + "created_at": "2026-07-14T11:39:06", + "label": "profile_eager_custom_ar_short_c1_t48_r1", + "url": "http://127.0.0.1:1111", + "model": "llm", + "prompt_mode": "short", + "with_tools": false, + "tool_count": 0, + "concurrency": 1, + "requests": 1, + "max_tokens": 48, + "wall_sec": 11.252297107130289, + "success_rate": 1.0, + "ttft_p50_sec": 4.141036370769143, + "ttft_p90_sec": 4.141036370769143, + "output_tps_p10_per_request": 6.7517276250194165, + "output_tps_p50_per_request": 6.7517276250194165, + "aggregate_output_tps": 4.265795645369481, + "prompt_tokens": 39, + "cached_tokens": 0, + "completion_tokens": 48, + "reasoning_tokens": 48, + "chars": 173, + "monitor": null, + "results": [ + { + "ok": true, + "elapsed_sec": 11.250327898189425, + "ttft_sec": 4.141036370769143, + "completion_tokens": 48, + "prompt_tokens": 39, + "cached_tokens": 0, + "reasoning_tokens": 48, + "output_tps": 6.7517276250194165, + "chars": 173, + "error": null + } + ] +} \ No newline at end of file diff --git a/worklogs/remote_results/2026-07-14-code-profile/profile_eager_custom_ar_short_c2_t32_r2.json b/worklogs/remote_results/2026-07-14-code-profile/profile_eager_custom_ar_short_c2_t32_r2.json new file mode 100644 index 0000000..c031b31 --- /dev/null +++ b/worklogs/remote_results/2026-07-14-code-profile/profile_eager_custom_ar_short_c2_t32_r2.json @@ -0,0 +1,51 @@ +{ + "created_at": "2026-07-14T11:39:16", + "label": "profile_eager_custom_ar_short_c2_t32_r2", + "url": "http://127.0.0.1:1111", + "model": "llm", + "prompt_mode": "short", + "with_tools": false, + "tool_count": 0, + "concurrency": 2, + "requests": 2, + "max_tokens": 32, + "wall_sec": 10.011093640699983, + "success_rate": 1.0, + "ttft_p50_sec": 1.5351156890392303, + "ttft_p90_sec": 1.5355193987488747, + "output_tps_p10_per_request": 3.776407383659353, + "output_tps_p50_per_request": 3.7764316482081455, + "aggregate_output_tps": 6.392907937631185, + "prompt_tokens": 78, + "cached_tokens": 64, + "completion_tokens": 64, + "reasoning_tokens": 64, + "chars": 210, + "monitor": null, + "results": [ + { + "ok": true, + "elapsed_sec": 10.00828673131764, + "ttft_sec": 1.534611051902175, + "completion_tokens": 32, + "prompt_tokens": 39, + "cached_tokens": 32, + "reasoning_tokens": 32, + "output_tps": 3.7764013175221547, + "chars": 105, + "error": null + }, + { + "ok": true, + "elapsed_sec": 10.00915989279747, + "ttft_sec": 1.5356203261762857, + "completion_tokens": 32, + "prompt_tokens": 39, + "cached_tokens": 32, + "reasoning_tokens": 32, + "output_tps": 3.7764619788941363, + "chars": 105, + "error": null + } + ] +} \ No newline at end of file diff --git a/worklogs/remote_results/2026-07-14-code-profile/profile_mode_eager_custom_ar_short_c1_t24_r1.json b/worklogs/remote_results/2026-07-14-code-profile/profile_mode_eager_custom_ar_short_c1_t24_r1.json new file mode 100644 index 0000000..25aa181 --- /dev/null +++ b/worklogs/remote_results/2026-07-14-code-profile/profile_mode_eager_custom_ar_short_c1_t24_r1.json @@ -0,0 +1,39 @@ +{ + "created_at": "2026-07-14T12:02:31", + "label": "profile_mode_eager_custom_ar_short_c1_t24_r1", + "url": "http://127.0.0.1:1111", + "model": "llm", + "prompt_mode": "short", + "with_tools": false, + "tool_count": 0, + "concurrency": 1, + "requests": 1, + "max_tokens": 24, + "wall_sec": 7.6276699639856815, + "success_rate": 1.0, + "ttft_p50_sec": 4.090665258467197, + "ttft_p90_sec": 4.090665258467197, + "output_tps_p10_per_request": 6.788945343504622, + "output_tps_p50_per_request": 6.788945343504622, + "aggregate_output_tps": 3.1464392289279512, + "prompt_tokens": 39, + "cached_tokens": 0, + "completion_tokens": 24, + "reasoning_tokens": 24, + "chars": 78, + "monitor": null, + "results": [ + { + "ok": true, + "elapsed_sec": 7.625824077054858, + "ttft_sec": 4.090665258467197, + "completion_tokens": 24, + "prompt_tokens": 39, + "cached_tokens": 0, + "reasoning_tokens": 24, + "output_tps": 6.788945343504622, + "chars": 78, + "error": null + } + ] +} \ No newline at end of file diff --git a/worklogs/remote_results/2026-07-14-code-profile/server_profile_eager_custom_ar_seq2_b8192.log b/worklogs/remote_results/2026-07-14-code-profile/server_profile_eager_custom_ar_seq2_b8192.log new file mode 100644 index 0000000..8323545 --- /dev/null +++ b/worklogs/remote_results/2026-07-14-code-profile/server_profile_eager_custom_ar_seq2_b8192.log @@ -0,0 +1,407 @@ +/usr/local/corex/lib/python3/dist-packages/torch/cuda/__init__.py:51: FutureWarning: The pynvml package is deprecated. Please install nvidia-ml-py instead. If you did not install pynvml directly, please report this to the maintainers of the package that installed pynvml for you. + import pynvml # type: ignore[import] +INFO 07-14 11:34:16 importing.py:10] Triton not installed; certain GPU-related functions will not be available. +2026-07-14 11:34:18.268441: I tensorflow/core/util/port.cc:110] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`. +2026-07-14 11:34:18.320614: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations. +To enable the following instructions: SSE3 SSE4.1 SSE4.2 AVX AVX2 AVX512F AVX512_VNNI AVX512_BF16 AVX_VNNI AMX_TILE AMX_INT8 AMX_BF16 FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags. +WARNING:tensorflow:Deprecation warnings have been disabled. Set TF_ENABLE_DEPRECATION_WARNINGS=1 to re-enable them. +INFO 07-14 11:34:23 api_server.py:530] vLLM API server version 0.6.3 +INFO 07-14 11:34:23 api_server.py:531] args: Namespace(host='0.0.0.0', port=1111, uvicorn_log_level='info', allow_credentials=False, allowed_origins=['*'], allowed_methods=['*'], allowed_headers=['*'], api_key=None, lora_modules=None, prompt_adapters=None, chat_template=None, response_role='assistant', ssl_keyfile=None, ssl_certfile=None, ssl_ca_certs=None, ssl_cert_reqs=0, root_path=None, middleware=[], return_tokens_as_token_ids=False, disable_frontend_multiprocessing=True, enable_auto_tool_choice=True, tool_call_parser='qwen3_coder', tool_parser_plugin='', reasoning_parser='qwen3', model='/root/public-storage/models/Qwen/Qwen3.6-35B-A3B', tokenizer=None, skip_tokenizer_init=False, revision=None, code_revision=None, tokenizer_revision=None, tokenizer_mode='auto', trust_remote_code=True, download_dir=None, load_format='auto', config_format='auto', dtype='auto', kv_cache_dtype='auto', quantization_param_path=None, max_model_len=100000, guided_decoding_backend='outlines', distributed_executor_backend=None, worker_use_ray=False, pipeline_parallel_size=1, tensor_parallel_size=4, max_parallel_loading_workers=None, ray_workers_use_nsight=False, block_size=16, enable_prefix_caching=True, disable_sliding_window=False, use_v2_block_manager=True, num_lookahead_slots=0, seed=0, swap_space=4, cpu_offload_gb=0, gpu_memory_utilization=0.95, num_gpu_blocks_override=None, max_num_batched_tokens=8192, max_num_seqs=2, max_logprobs=20, disable_log_stats=False, quantization=None, rope_scaling=None, rope_theta=None, enforce_eager=True, max_context_len_to_capture=None, max_seq_len_to_capture=32768, disable_custom_all_reduce=False, tokenizer_pool_size=0, tokenizer_pool_type='ray', tokenizer_pool_extra_config=None, limit_mm_per_prompt=None, mm_processor_kwargs=None, enable_lora=False, max_loras=1, max_lora_rank=16, lora_extra_vocab_size=256, lora_dtype='auto', long_lora_scaling_factors=None, max_cpu_loras=None, fully_sharded_loras=False, enable_prompt_adapter=False, max_prompt_adapters=1, max_prompt_adapter_token=0, device='auto', num_scheduler_steps=1, multi_step_stream_outputs=True, scheduler_delay_factor=0.0, enable_chunked_prefill=True, speculative_model=None, speculative_model_quantization=None, num_speculative_tokens=None, speculative_disable_mqa_scorer=False, speculative_draft_tensor_parallel_size=None, speculative_max_model_len=None, speculative_disable_by_batch_size=None, ngram_prompt_lookup_max=None, ngram_prompt_lookup_min=None, spec_decoding_acceptance_method='rejection_sampler', typical_acceptance_sampler_posterior_threshold=None, typical_acceptance_sampler_posterior_alpha=None, disable_logprobs_during_spec_decoding=None, model_loader_extra_config=None, ignore_patterns=[], preemption_mode=None, served_model_name=['llm'], qlora_adapter_name_or_path=None, otlp_traces_endpoint=None, collect_detailed_traces=None, disable_async_output_proc=False, override_neuron_config=None, scheduling_policy='fcfs', disable_log_requests=True, max_log_len=None, disable_fastapi_docs=False) +INFO 07-14 11:34:23 config.py:1670] Downcasting torch.float32 to torch.float16. +INFO 07-14 11:34:34 config.py:887] Defaulting to use mp for distributed inference +INFO 07-14 11:34:34 config.py:1005] Chunked prefill is enabled with max_num_batched_tokens=8192. +WARNING 07-14 11:34:34 config.py:380] To see benefits of async output processing, enable CUDA graph. Since, enforce-eager is enabled, async output processor cannot be used +INFO 07-14 11:34:34 llm_engine.py:237] Initializing an LLM engine (v0.6.3) with config: model='/root/public-storage/models/Qwen/Qwen3.6-35B-A3B', speculative_config=None, tokenizer='/root/public-storage/models/Qwen/Qwen3.6-35B-A3B', skip_tokenizer_init=False, tokenizer_mode=auto, revision=None, override_neuron_config=None, rope_scaling=None, rope_theta=None, tokenizer_revision=None, trust_remote_code=True, dtype=torch.float16, max_seq_len=100000, download_dir=None, load_format=LoadFormat.AUTO, tensor_parallel_size=4, pipeline_parallel_size=1, disable_custom_all_reduce=False, quantization=None, enforce_eager=True, kv_cache_dtype=auto, quantization_param_path=None, device_config=cuda, decoding_config=DecodingConfig(guided_decoding_backend='outlines'), observability_config=ObservabilityConfig(otlp_traces_endpoint=None, collect_model_forward_time=False, collect_model_execute_time=False), seed=0, served_model_name=llm, use_v2_block_manager=True, num_scheduler_steps=1, chunked_prefill_enabled=True multi_step_stream_outputs=True, enable_prefix_caching=True, use_async_output_proc=False, use_cached_outputs=False, mm_processor_kwargs=None) +WARNING 07-14 11:34:35 multiproc_gpu_executor.py:53] Reducing Torch parallelism from 64 threads to 1 to avoid unnecessary CPU contention. Set OMP_NUM_THREADS in the external environment to tune this value as needed. +INFO 07-14 11:34:35 custom_cache_manager.py:17] Setting Triton cache manager to: vllm.triton_utils.custom_cache_manager:CustomCacheManager +INFO 07-14 11:34:35 selector.py:266] Cannot use FlashAttention-2 backend because the vllm.vllm_flash_attn package is not found. Make sure that vllm_flash_attn was built and installed (on by default). +INFO 07-14 11:34:35 selector.py:115] Using XFormers backend. +/usr/local/corex/lib/python3/dist-packages/torch/cuda/__init__.py:51: FutureWarning: The pynvml package is deprecated. Please install nvidia-ml-py instead. If you did not install pynvml directly, please report this to the maintainers of the package that installed pynvml for you. + import pynvml # type: ignore[import] +/usr/local/corex/lib/python3/dist-packages/torch/cuda/__init__.py:51: FutureWarning: The pynvml package is deprecated. Please install nvidia-ml-py instead. If you did not install pynvml directly, please report this to the maintainers of the package that installed pynvml for you. + import pynvml # type: ignore[import] +/usr/local/corex/lib/python3/dist-packages/torch/cuda/__init__.py:51: FutureWarning: The pynvml package is deprecated. Please install nvidia-ml-py instead. If you did not install pynvml directly, please report this to the maintainers of the package that installed pynvml for you. + import pynvml # type: ignore[import] +INFO 07-14 11:34:37 importing.py:10] Triton not installed; certain GPU-related functions will not be available. +INFO 07-14 11:34:37 importing.py:10] Triton not installed; certain GPU-related functions will not be available. +INFO 07-14 11:34:37 importing.py:10] Triton not installed; certain GPU-related functions will not be available. +WARNING:tensorflow:Deprecation warnings have been disabled. Set TF_ENABLE_DEPRECATION_WARNINGS=1 to re-enable them. +WARNING:tensorflow:Deprecation warnings have been disabled. Set TF_ENABLE_DEPRECATION_WARNINGS=1 to re-enable them. +WARNING:tensorflow:Deprecation warnings have been disabled. Set TF_ENABLE_DEPRECATION_WARNINGS=1 to re-enable them. +(VllmWorkerProcess pid=10987) INFO 07-14 11:34:44 selector.py:266] Cannot use FlashAttention-2 backend because the vllm.vllm_flash_attn package is not found. Make sure that vllm_flash_attn was built and installed (on by default). +(VllmWorkerProcess pid=10987) INFO 07-14 11:34:44 selector.py:115] Using XFormers backend. +(VllmWorkerProcess pid=10987) INFO 07-14 11:34:44 multiproc_worker_utils.py:216] Worker ready; awaiting tasks +(VllmWorkerProcess pid=10988) INFO 07-14 11:34:44 selector.py:266] Cannot use FlashAttention-2 backend because the vllm.vllm_flash_attn package is not found. Make sure that vllm_flash_attn was built and installed (on by default). +(VllmWorkerProcess pid=10988) INFO 07-14 11:34:44 selector.py:115] Using XFormers backend. +(VllmWorkerProcess pid=10988) INFO 07-14 11:34:44 multiproc_worker_utils.py:216] Worker ready; awaiting tasks +(VllmWorkerProcess pid=10989) INFO 07-14 11:34:44 selector.py:266] Cannot use FlashAttention-2 backend because the vllm.vllm_flash_attn package is not found. Make sure that vllm_flash_attn was built and installed (on by default). +(VllmWorkerProcess pid=10989) INFO 07-14 11:34:44 selector.py:115] Using XFormers backend. +(VllmWorkerProcess pid=10989) INFO 07-14 11:34:44 multiproc_worker_utils.py:216] Worker ready; awaiting tasks +INFO 07-14 11:34:45 shm_broadcast.py:242] vLLM message queue communication handle: Handle(connect_ip='127.0.0.1', local_reader_ranks=[1, 2, 3], buffer=, local_subscribe_port=42265, remote_subscribe_port=None) +INFO 07-14 11:34:45 model_runner.py:1112] Starting to load model /root/public-storage/models/Qwen/Qwen3.6-35B-A3B... +(VllmWorkerProcess pid=10987) INFO 07-14 11:34:45 model_runner.py:1112] Starting to load model /root/public-storage/models/Qwen/Qwen3.6-35B-A3B... +(VllmWorkerProcess pid=10988) INFO 07-14 11:34:45 model_runner.py:1112] Starting to load model /root/public-storage/models/Qwen/Qwen3.6-35B-A3B... +(VllmWorkerProcess pid=10989) INFO 07-14 11:34:45 model_runner.py:1112] Starting to load model /root/public-storage/models/Qwen/Qwen3.6-35B-A3B... +INFO 07-14 11:34:45 selector.py:266] Cannot use FlashAttention-2 backend because the vllm.vllm_flash_attn package is not found. Make sure that vllm_flash_attn was built and installed (on by default). +INFO 07-14 11:34:45 selector.py:115] Using XFormers backend. +(VllmWorkerProcess pid=10988) INFO 07-14 11:34:45 selector.py:266] Cannot use FlashAttention-2 backend because the vllm.vllm_flash_attn package is not found. Make sure that vllm_flash_attn was built and installed (on by default). +(VllmWorkerProcess pid=10987) INFO 07-14 11:34:45 selector.py:266] Cannot use FlashAttention-2 backend because the vllm.vllm_flash_attn package is not found. Make sure that vllm_flash_attn was built and installed (on by default). +(VllmWorkerProcess pid=10988) INFO 07-14 11:34:45 selector.py:115] Using XFormers backend. +(VllmWorkerProcess pid=10987) INFO 07-14 11:34:45 selector.py:115] Using XFormers backend. +(VllmWorkerProcess pid=10989) INFO 07-14 11:34:45 selector.py:266] Cannot use FlashAttention-2 backend because the vllm.vllm_flash_attn package is not found. Make sure that vllm_flash_attn was built and installed (on by default). +(VllmWorkerProcess pid=10989) INFO 07-14 11:34:45 selector.py:115] Using XFormers backend. + Loading safetensors checkpoint shards: 0% Completed | 0/26 [00:00, local_subscribe_port=60007, remote_subscribe_port=None) +INFO 07-14 11:58:07 model_runner.py:1112] Starting to load model /root/public-storage/models/Qwen/Qwen3.6-35B-A3B... +(VllmWorkerProcess pid=12140) INFO 07-14 11:58:07 model_runner.py:1112] Starting to load model /root/public-storage/models/Qwen/Qwen3.6-35B-A3B... +(VllmWorkerProcess pid=12141) INFO 07-14 11:58:07 model_runner.py:1112] Starting to load model /root/public-storage/models/Qwen/Qwen3.6-35B-A3B... +(VllmWorkerProcess pid=12142) INFO 07-14 11:58:07 model_runner.py:1112] Starting to load model /root/public-storage/models/Qwen/Qwen3.6-35B-A3B... +INFO 07-14 11:58:07 selector.py:266] Cannot use FlashAttention-2 backend because the vllm.vllm_flash_attn package is not found. Make sure that vllm_flash_attn was built and installed (on by default). +INFO 07-14 11:58:07 selector.py:115] Using XFormers backend. +(VllmWorkerProcess pid=12142) INFO 07-14 11:58:07 selector.py:266] Cannot use FlashAttention-2 backend because the vllm.vllm_flash_attn package is not found. Make sure that vllm_flash_attn was built and installed (on by default). +(VllmWorkerProcess pid=12140) INFO 07-14 11:58:07 selector.py:266] Cannot use FlashAttention-2 backend because the vllm.vllm_flash_attn package is not found. Make sure that vllm_flash_attn was built and installed (on by default). +(VllmWorkerProcess pid=12141) INFO 07-14 11:58:07 selector.py:266] Cannot use FlashAttention-2 backend because the vllm.vllm_flash_attn package is not found. Make sure that vllm_flash_attn was built and installed (on by default). +(VllmWorkerProcess pid=12142) INFO 07-14 11:58:07 selector.py:115] Using XFormers backend. +(VllmWorkerProcess pid=12140) INFO 07-14 11:58:07 selector.py:115] Using XFormers backend. +(VllmWorkerProcess pid=12141) INFO 07-14 11:58:07 selector.py:115] Using XFormers backend. + Loading safetensors checkpoint shards: 0% Completed | 0/26 [00:00, local_subscribe_port=52711, remote_subscribe_port=None) +INFO 07-14 12:25:36 model_runner.py:1112] Starting to load model /root/public-storage/models/Qwen/Qwen3.6-35B-A3B... +(VllmWorkerProcess pid=14380) INFO 07-14 12:25:36 model_runner.py:1112] Starting to load model /root/public-storage/models/Qwen/Qwen3.6-35B-A3B... +(VllmWorkerProcess pid=14381) INFO 07-14 12:25:36 model_runner.py:1112] Starting to load model /root/public-storage/models/Qwen/Qwen3.6-35B-A3B... +(VllmWorkerProcess pid=14382) INFO 07-14 12:25:36 model_runner.py:1112] Starting to load model /root/public-storage/models/Qwen/Qwen3.6-35B-A3B... +INFO 07-14 12:25:36 selector.py:266] Cannot use FlashAttention-2 backend because the vllm.vllm_flash_attn package is not found. Make sure that vllm_flash_attn was built and installed (on by default). +INFO 07-14 12:25:36 selector.py:115] Using XFormers backend. +(VllmWorkerProcess pid=14380) INFO 07-14 12:25:36 selector.py:266] Cannot use FlashAttention-2 backend because the vllm.vllm_flash_attn package is not found. Make sure that vllm_flash_attn was built and installed (on by default). +(VllmWorkerProcess pid=14382) INFO 07-14 12:25:36 selector.py:266] Cannot use FlashAttention-2 backend because the vllm.vllm_flash_attn package is not found. Make sure that vllm_flash_attn was built and installed (on by default). +(VllmWorkerProcess pid=14380) INFO 07-14 12:25:36 selector.py:115] Using XFormers backend. +(VllmWorkerProcess pid=14382) INFO 07-14 12:25:36 selector.py:115] Using XFormers backend. +(VllmWorkerProcess pid=14381) INFO 07-14 12:25:36 selector.py:266] Cannot use FlashAttention-2 backend because the vllm.vllm_flash_attn package is not found. Make sure that vllm_flash_attn was built and installed (on by default). +(VllmWorkerProcess pid=14381) INFO 07-14 12:25:36 selector.py:115] Using XFormers backend. + Loading safetensors checkpoint shards: 0% Completed | 0/26 [00:00