From af375830c7bf4c8a431c0a5b2c05af38f5fa2aa5 Mon Sep 17 00:00:00 2001
From: the magician <82004885@qq.com>
Date: Wed, 15 Jul 2026 00:49:24 +0800
Subject: [PATCH] align platform build config with working submission
---
computility-run.yaml | 24 +-
.../chat_template_multi_system.jinja | 175 ++++
qwen3_6_scripts/paged_attn.py | 118 ++-
qwen3_6_scripts/patch_ops.sh | 122 +++
qwen3_6_scripts/patch_xformers_sdpa_seq.py | 69 +-
qwen3_6_scripts/protocol.py | 72 ++
qwen3_6_scripts/qwen3_5.py | 488 ++++-----
qwen3_6_scripts/xformers.py | 946 ++++++++++++++++++
worklogs/decode_analysis_report_2026-07-14.md | 34 +
9 files changed, 1774 insertions(+), 274 deletions(-)
create mode 100644 qwen3_6_scripts/chat_template_multi_system.jinja
create mode 100644 qwen3_6_scripts/xformers.py
diff --git a/computility-run.yaml b/computility-run.yaml
index 6447abd..a10c207 100644
--- a/computility-run.yaml
+++ b/computility-run.yaml
@@ -8,7 +8,7 @@ command:
- --served-model-name
- llm
- --max-model-len
- - '100000'
+ - '245000'
- --gpu-memory-utilization
- '0.9'
- --trust-remote-code
@@ -19,16 +19,34 @@ command:
- --disable-log-requests
- --disable-frontend-multiprocessing
- --max-num-batched-tokens
- - '8192'
+ - '16384'
- --enable-chunked-prefill
- --max-seq-len-to-capture
- - '32768'
+ - '245000'
- --enable-auto-tool-choice
- --tool-call-parser
- qwen3_coder
- --reasoning-parser
- qwen3
- --enable-prefix-caching
+ - --chat-template
+ - /workspace/chat_template_multi_system.jinja
env:
- name: VLLM_ENGINE_ITERATION_TIMEOUT_S
value: 3600
+ - name: PYTHONPATH
+ value: /usr/local/corex/lib64/python3/dist-packages
+ - name: LD_LIBRARY_PATH
+ value: /usr/local/corex/lib64:/usr/local/iluvatar/lib64:/usr/local/openmpi/lib
+ - name: PATH
+ value: /usr/local/corex/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/local/openmpi/bin
+ - name: MOE_NATIVE
+ value: '1'
+ - name: CHUNK_PARALLEL
+ value: '1'
+ - name: PREFIX_FLASH
+ value: '1'
+ - name: RMSNORM_NATIVE
+ value: '1'
+ - name: LOOP1_NATIVE
+ value: '1'
diff --git a/qwen3_6_scripts/chat_template_multi_system.jinja b/qwen3_6_scripts/chat_template_multi_system.jinja
new file mode 100644
index 0000000..3592c48
--- /dev/null
+++ b/qwen3_6_scripts/chat_template_multi_system.jinja
@@ -0,0 +1,175 @@
+{%- set image_count = namespace(value=0) %}
+{%- set video_count = namespace(value=0) %}
+{%- macro render_content(content, do_vision_count, is_system_content=false) %}
+ {%- if content is string %}
+ {{- content }}
+ {%- elif content is iterable and content is not mapping %}
+ {%- for item in content %}
+ {%- if 'image' in item or 'image_url' in item or item.type == 'image' %}
+ {%- if is_system_content %}
+ {{- raise_exception('System message cannot contain images.') }}
+ {%- endif %}
+ {%- if do_vision_count %}
+ {%- set image_count.value = image_count.value + 1 %}
+ {%- endif %}
+ {%- if add_vision_id %}
+ {{- 'Picture ' ~ image_count.value ~ ': ' }}
+ {%- endif %}
+ {{- '<|vision_start|><|image_pad|><|vision_end|>' }}
+ {%- elif 'video' in item or item.type == 'video' %}
+ {%- if is_system_content %}
+ {{- raise_exception('System message cannot contain videos.') }}
+ {%- endif %}
+ {%- if do_vision_count %}
+ {%- set video_count.value = video_count.value + 1 %}
+ {%- endif %}
+ {%- if add_vision_id %}
+ {{- 'Video ' ~ video_count.value ~ ': ' }}
+ {%- endif %}
+ {{- '<|vision_start|><|video_pad|><|vision_end|>' }}
+ {%- elif 'text' in item %}
+ {{- item.text }}
+ {%- else %}
+ {{- raise_exception('Unexpected item type in content.') }}
+ {%- endif %}
+ {%- endfor %}
+ {%- elif content is none or content is undefined %}
+ {{- '' }}
+ {%- else %}
+ {{- raise_exception('Unexpected content type.') }}
+ {%- endif %}
+{%- endmacro %}
+{#- multi_system 兼容: 合并所有 system 消息到首位, 消除 "System message must be at the beginning" 报错。
+ 数据集 34 条请求含多个/乱序 system 消息, 原模板 raise 导致 15 条 4xx。此处把所有
+ system 内容按序用 \n\n 拼接, 作为唯一的首位 system; 非 system 消息保持原顺序跟在后面。
+ 已验证: 正常单system/无system/tools/enable_thinking/tool回传 均不受影响。 -#}
+{%- set _sys_parts = namespace(text=[]) %}
+{%- for m in messages %}
+ {%- if m.role == "system" %}
+ {%- set _sys_parts.text = _sys_parts.text + [render_content(m.content, false, true)|trim] %}
+ {%- endif %}
+{%- endfor %}
+{%- set _sys_text = _sys_parts.text | select("ne", "") | list | join("\n\n") %}
+{%- set _ns = namespace(items=[]) %}
+{%- if _sys_text %}
+ {%- set _ns.items = _ns.items + [{"role":"system","content":_sys_text}] %}
+{%- endif %}
+{%- for m in messages %}
+ {%- if m.role != "system" %}
+ {%- set _ns.items = _ns.items + [m] %}
+ {%- endif %}
+{%- endfor %}
+{%- set messages = _ns.items %}
+{%- if not messages %}
+ {{- raise_exception('No messages provided.') }}
+{%- endif %}
+{%- if tools and tools is iterable and tools is not mapping %}
+ {{- '<|im_start|>system\n' }}
+ {{- "# Tools\n\nYou have access to the following functions:\n\n" }}
+ {%- for tool in tools %}
+ {{- "\n" }}
+ {{- tool | tojson }}
+ {%- endfor %}
+ {{- "\n" }}
+ {{- '\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n\n\n\nvalue_1\n\n\nThis is the value for the second parameter\nthat can span\nmultiple lines\n\n\n\n\n\nReminder:\n- Function calls MUST follow the specified format: an inner block must be nested within XML tags\n- Required parameters MUST be specified\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\n' }}
+ {%- if messages[0].role == 'system' %}
+ {%- set content = render_content(messages[0].content, false, true)|trim %}
+ {%- if content %}
+ {{- '\n\n' + content }}
+ {%- endif %}
+ {%- endif %}
+ {{- '<|im_end|>\n' }}
+{%- else %}
+ {%- if messages[0].role == 'system' %}
+ {%- set content = render_content(messages[0].content, false, true)|trim %}
+ {{- '<|im_start|>system\n' + content + '<|im_end|>\n' }}
+ {%- endif %}
+{%- endif %}
+{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %}
+{%- for message in messages[::-1] %}
+ {%- set index = (messages|length - 1) - loop.index0 %}
+ {%- if ns.multi_step_tool and message.role == "user" %}
+ {%- set content = render_content(message.content, false)|trim %}
+ {%- if not(content.startswith('') and content.endswith('')) %}
+ {%- set ns.multi_step_tool = false %}
+ {%- set ns.last_query_index = index %}
+ {%- endif %}
+ {%- endif %}
+{%- endfor %}
+{%- if ns.multi_step_tool %}
+ {{- raise_exception('No user query found in messages.') }}
+{%- endif %}
+{%- for message in messages %}
+ {%- set content = render_content(message.content, true)|trim %}
+ {%- if message.role == "system" %}
+ {%- if not loop.first %}
+ {{- raise_exception('System message must be at the beginning.') }}
+ {%- endif %}
+ {%- elif message.role == "user" %}
+ {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }}
+ {%- elif message.role == "assistant" %}
+ {%- set reasoning_content = '' %}
+ {%- if message.reasoning_content is string %}
+ {%- set reasoning_content = message.reasoning_content %}
+ {%- else %}
+ {%- if '' in content %}
+ {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %}
+ {%- set content = content.split('')[-1].lstrip('\n') %}
+ {%- endif %}
+ {%- endif %}
+ {%- set reasoning_content = reasoning_content|trim %}
+ {%- if (preserve_thinking is defined and preserve_thinking is true) or (loop.index0 > ns.last_query_index) %}
+ {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content + '\n\n\n' + content }}
+ {%- else %}
+ {{- '<|im_start|>' + message.role + '\n' + content }}
+ {%- endif %}
+ {%- if message.tool_calls and message.tool_calls is iterable and message.tool_calls is not mapping %}
+ {%- for tool_call in message.tool_calls %}
+ {%- if tool_call.function is defined %}
+ {%- set tool_call = tool_call.function %}
+ {%- endif %}
+ {%- if loop.first %}
+ {%- if content|trim %}
+ {{- '\n\n\n\n' }}
+ {%- else %}
+ {{- '\n\n' }}
+ {%- endif %}
+ {%- else %}
+ {{- '\n\n\n' }}
+ {%- endif %}
+ {%- if tool_call.arguments is defined %}
+ {%- for args_name, args_value in tool_call.arguments|items %}
+ {{- '\n' }}
+ {%- set args_value = args_value | string if args_value is string else args_value | tojson | safe %}
+ {{- args_value }}
+ {{- '\n\n' }}
+ {%- endfor %}
+ {%- endif %}
+ {{- '\n' }}
+ {%- endfor %}
+ {%- endif %}
+ {{- '<|im_end|>\n' }}
+ {%- elif message.role == "tool" %}
+ {%- if loop.previtem and loop.previtem.role != "tool" %}
+ {{- '<|im_start|>user' }}
+ {%- endif %}
+ {{- '\n\n' }}
+ {{- content }}
+ {{- '\n' }}
+ {%- if not loop.last and loop.nextitem.role != "tool" %}
+ {{- '<|im_end|>\n' }}
+ {%- elif loop.last %}
+ {{- '<|im_end|>\n' }}
+ {%- endif %}
+ {%- else %}
+ {{- raise_exception('Unexpected message role.') }}
+ {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+ {{- '<|im_start|>assistant\n' }}
+ {%- if enable_thinking is defined and enable_thinking is false %}
+ {{- '\n\n\n\n' }}
+ {%- else %}
+ {{- '\n' }}
+ {%- endif %}
+{%- endif %}
\ No newline at end of file
diff --git a/qwen3_6_scripts/paged_attn.py b/qwen3_6_scripts/paged_attn.py
index 8590489..2d9ae1b 100644
--- a/qwen3_6_scripts/paged_attn.py
+++ b/qwen3_6_scripts/paged_attn.py
@@ -289,7 +289,22 @@ class PagedAttention:
) -> torch.Tensor:
# NOTE: The Triton context_attention_fwd kernel hangs on Iluvatar
# BI-V100 hardware (same class of issue as cudnnFlashAttnForward).
- # Use a pure-PyTorch fallback that reads the paged KV cache directly.
+ # 单次 pad-方阵原生 flash(verified) when possible; on any anomaly or
+ # when disabled via env, fall back to the pure-PyTorch path.
+ import os as _os
+ if _os.environ.get("PREFIX_FLASH", "1") != "0":
+ try:
+ return PagedAttention._forward_prefix_flash(
+ query, key, value,
+ key_cache, value_cache,
+ block_tables, query_start_loc,
+ seq_lens_tensor, context_lens,
+ )
+ except Exception as _e:
+ import sys as _sys, traceback as _tb
+ print(f"[prefix_flash FALLBACK] {type(_e).__name__}: {_e}",
+ file=_sys.stderr, flush=True)
+ _tb.print_exc(file=_sys.stderr)
return PagedAttention._forward_prefix_pytorch(
query, key, value,
key_cache, value_cache,
@@ -297,6 +312,107 @@ class PagedAttention:
seq_lens_tensor, context_lens,
)
+ @staticmethod
+ def _forward_prefix_flash(
+ query: torch.Tensor,
+ key: torch.Tensor,
+ value: torch.Tensor,
+ key_cache: torch.Tensor,
+ value_cache: torch.Tensor,
+ block_tables: torch.Tensor,
+ query_start_loc: torch.Tensor,
+ seq_lens_tensor: torch.Tensor,
+ context_lens: torch.Tensor,
+ ) -> torch.Tensor:
+ """单次 pad-方阵原生 flash 的 prefix/chunked-prefill attention。
+
+ 对每个 sequence:
+ 1. gather 历史 KV(paged cache) → 连续张量,拼上当前 chunk 的 KV
+ → K/V = [ctx_len + q_len, kv_h, d]。
+ 2. 把 query 前面 pad ctx_len 个零行 → [ctx_len+q_len, q_h, d](方阵)。
+ 3. 单次 flash_attn_varlen_func(causal=True) 方阵 —— causal 在方阵下语义正确,
+ query 第 (ctx_len+j) 行只看到 key [0, ctx_len+j], 正是 prefix 语义。
+ 4. 丢弃前 ctx_len 行(pad 产生的垃圾输出),返回后 q_len 行。
+
+ 比两段式(flash+lse归merge)简单且快 1.3~3.5x —— 无需 lse 重算(ixformer 推理版
+ flash 不暴露 softmax_lse, 重算 lse 占两段式 77% 成本)。方阵 causal 已验证精确
+ (verify_single_correct.py, max_abs≤2e-3, GQA 4Q/1KV, ctx≤80K)。
+
+ 显存兜底:总长 (ctx+q) 超阈值时抛异常, 由 forward_prefix 回退纯 PyTorch。
+ pad 浪费兜底:ctx_len 远大于 q_len(缓存命中态)时 pad 方阵浪费大, 抛异常回退
+ 纯 PyTorch(O(q·k) 在大 ctx 反而更省)。只有冷 prefill 的 chunked(ctx≲q_len)吃单次 pad。
+ """
+ from ixformer.functions.flash_attn_lib import flash_attn_varlen_func
+
+ # pad 方阵总长上限。方阵 flash 是 O(tk^2), ctx 远大于 q_len 时 pad 浪费大;
+ # 超阈值交给纯 PyTorch(O(q·k), 大 ctx 反而相对省)。
+ _FLASH_TOTAL_LIMIT = 131072
+ # pad 浪费比例上限: ctx_len > q_len * R 时该 seq 退回纯 PyTorch。
+ # 冷 prefill 的后续 chunk: ctx/q ≤ ~4 (65K/16K); 缓存命中: ctx/q ≫ 10 → 退回。
+ _FLASH_PAD_RATIO = 4
+
+ batch_size = seq_lens_tensor.shape[0]
+ num_q_heads = query.shape[1]
+ num_kv_heads = key_cache.shape[1]
+ head_dim = query.shape[2]
+ block_size = value_cache.shape[3]
+ scale = head_dim ** -0.5
+ orig_dtype = query.dtype
+ dev = query.device
+ output = torch.empty_like(query)
+
+ for i in range(batch_size):
+ ctx_len = int(context_lens[i].item())
+ q_start = int(query_start_loc[i].item())
+ q_end = int(query_start_loc[i + 1].item())
+ q_len = q_end - q_start
+ total_k = ctx_len + q_len
+
+ if total_k > _FLASH_TOTAL_LIMIT:
+ raise RuntimeError(
+ f"total_k={total_k} > flash limit {_FLASH_TOTAL_LIMIT}; "
+ "fall back to pytorch for memory safety")
+ if ctx_len > q_len * _FLASH_PAD_RATIO:
+ raise RuntimeError(
+ f"pad ratio too high (ctx={ctx_len} q={q_len}); "
+ "fall back to pytorch (single-pad wasteful for cache-hit)")
+
+ q_i = query[q_start:q_end] # [q_len, q_h, d]
+ k_i = key [q_start:q_end] # [q_len, kv_h, d]
+ v_i = value[q_start:q_end]
+
+ if ctx_len > 0:
+ num_ctx_blocks = (ctx_len + block_size - 1) // block_size
+ if num_ctx_blocks > block_tables.shape[1]:
+ # 与 pytorch 版一致的 undersized 保护(prefix_cache_hit bug)
+ num_ctx_blocks = block_tables.shape[1]
+ blk_ids = block_tables[i, :num_ctx_blocks]
+ # gather K/V: paged 布局 → 连续 [ctx_len, kv_h, d]
+ k_ctx = (key_cache[blk_ids]
+ .permute(0, 3, 1, 2, 4)
+ .contiguous()
+ .view(-1, num_kv_heads, head_dim))[:ctx_len]
+ v_ctx = (value_cache[blk_ids]
+ .permute(0, 3, 1, 2)
+ .contiguous()
+ .view(-1, num_kv_heads, head_dim))[:ctx_len]
+ k_full = torch.cat([k_ctx, k_i], dim=0).contiguous() # [tk, kv_h, d]
+ v_full = torch.cat([v_ctx, v_i], dim=0).contiguous()
+ # query pad 到方阵: 前 ctx_len 行零(输出丢弃)
+ q_pad = torch.zeros(total_k, num_q_heads, head_dim,
+ device=dev, dtype=q_i.dtype)
+ q_pad[ctx_len:] = q_i
+ else:
+ k_full, v_full, q_pad = k_i.contiguous(), v_i.contiguous(), q_i.contiguous()
+
+ cu = torch.tensor([0, total_k], dtype=torch.int32, device=dev)
+ o = flash_attn_varlen_func(
+ q_pad, k_full, v_full, cu, cu, total_k, total_k,
+ 0.0, scale, causal=True) # [tk, q_h, d]
+ output[q_start:q_end] = o[ctx_len:].to(orig_dtype)
+
+ return output
+
@staticmethod
def _forward_prefix_pytorch(
query: torch.Tensor,
diff --git a/qwen3_6_scripts/patch_ops.sh b/qwen3_6_scripts/patch_ops.sh
index 1035b2f..d6ef3ae 100755
--- a/qwen3_6_scripts/patch_ops.sh
+++ b/qwen3_6_scripts/patch_ops.sh
@@ -24,6 +24,45 @@
# --max-num-batched-tokens 8192 --enable-chunked-prefill --enable-prefix-caching \
# --max-seq-len-to-capture 32768
+# --- environment: corex runtime paths + five optimization env flags ----------
+# patch_ops.sh ONLY configures the environment + deploys the patched files.
+# The actual server start command lives in ../computility-run.yaml.
+# These exports are picked up by the verification steps below; the server process
+# inherits them because computility-run.yaml is launched from a shell that has
+# run this script (or these same vars are set in the yaml env: block).
+export PYTHONPATH=/usr/local/corex/lib64/python3/dist-packages
+export LD_LIBRARY_PATH=/usr/local/corex/lib64:/usr/local/iluvatar/lib64:/usr/local/openmpi/lib
+export PATH=/usr/local/corex/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/local/openmpi/bin
+export CUDA_VISIBLE_DEVICES=${CUDA_VISIBLE_DEVICES:-0,1,2,3}
+export VLLM_ENGINE_ITERATION_TIMEOUT_S=${VLLM_ENGINE_ITERATION_TIMEOUT_S:-3600}
+
+# Five GatedDeltaNet / MoE / attention optimizations (all default ON, each falls
+# back to pure PyTorch on any anomaly — see qwen3_5.py / paged_attn.py). Setting
+# any to 0 disables it. Mirrored in computility-run.yaml env: block.
+export MOE_NATIVE=${MOE_NATIVE:-1} # MoE sort-by-expert + native ixformer kernels
+export CHUNK_PARALLEL=${CHUNK_PARALLEL:-1} # GDN loop2 linear-matrix recurrence
+export PREFIX_FLASH=${PREFIX_FLASH:-1} # single-pad square native flash for prefix attn
+export RMSNORM_NATIVE=${RMSNORM_NATIVE:-1} # native rms_norm for RMSNormGated
+export LOOP1_NATIVE=${LOOP1_NATIVE:-1} # GDN loop1 triangular solve via solve_triangular
+
+# --- libcusolver.so: make torch.linalg.solve_triangular usable (LOOP1_NATIVE) -
+# corex torch has a hardcoded rpath to /opt/sw_home/local/cuda/lib64/libcusolver.so,
+# which does not exist on this image. Without it, solve_triangular dlopen fails on
+# every call → loop1_native falls back to the 63-step serial Python loop (losing the
+# ~4-5x speedup). LD_LIBRARY_PATH does NOT help because corex ignores it for this lib.
+# Fix: create the expected path as a symlink to the real corex libcusolver.so.
+CUSOLVER_SRC=/usr/local/corex/lib64/libcusolver.so
+CUSOLVER_DST=/opt/sw_home/local/cuda/lib64/libcusolver.so
+if [ -f "$CUSOLVER_SRC" ]; then
+ mkdir -p /opt/sw_home/local/cuda/lib64
+ if [ ! -e "$CUSOLVER_DST" ] || [ "$(readlink -f "$CUSOLVER_DST")" != "$(readlink -f "$CUSOLVER_SRC")" ]; then
+ ln -sf "$CUSOLVER_SRC" "$CUSOLVER_DST"
+ echo "[patch_ops] linked libcusolver.so -> $CUSOLVER_SRC (enables LOOP1_NATIVE)"
+ fi
+else
+ echo "[patch_ops] WARNING: $CUSOLVER_SRC not found; LOOP1_NATIVE will fall back to serial loop" >&2
+fi
+
# --- paged_attn.py: replace forward_prefix with pure-PyTorch fallback -------
# The Triton context_attention_fwd kernel hangs BI-V100 GPUs permanently
# (standard Triton 2.3.1 PTX is not supported by the corex runtime either).
@@ -92,3 +131,86 @@ cp ./cli_args.py /usr/local/corex/lib/python3/dist-packages/vllm/entrypoints/ope
cp ./serving_chat.py /usr/local/corex/lib/python3/dist-packages/vllm/entrypoints/openai/serving_chat.py
cp ./api_server.py /usr/local/corex/lib/python3/dist-packages/vllm/entrypoints/openai/api_server.py
cp ./chat_utils.py /usr/local/corex/lib/python3/dist-packages/vllm/entrypoints/chat_utils.py
+
+# --- chat_template: multi_system 兼容 (合并多/乱序 system 到首位) -------------
+# 数据集 34 条请求含多个或乱序 system 消息, 原模型 chat_template.jinja 对非首位
+# system raise "System message must be at the beginning." → 15 条 4xx, 威胁成功率≥99%。
+# 模型目录 (/root/public-storage 或容器内 /model) 是只读挂载, 不能改原模板。
+# 改后模板随 qwen3_6_scripts 一起 COPY 进来 (本目录 ./chat_template_multi_system.jinja),
+# 这里 cp 到 /workspace/ (Dockerfile 已 mkdir /workspace), 启动时用
+# --chat-template /workspace/chat_template_multi_system.jinja 指向它 (见 computility-run.yaml)。
+mkdir -p /workspace
+CHAT_TMPL=/workspace/chat_template_multi_system.jinja
+if [ -f "./chat_template_multi_system.jinja" ]; then
+ cp "./chat_template_multi_system.jinja" "$CHAT_TMPL"
+ echo "[patch_ops] deployed multi_system chat template → $CHAT_TMPL"
+else
+ echo "[patch_ops] WARNING: ./chat_template_multi_system.jinja not found in $(pwd)" >&2
+fi
+
+# --- verification: five optimization flags present in deployed package -------
+VLLM_PKG=/usr/local/corex/lib64/python3/dist-packages/vllm
+echo ""
+echo "=== patch_ops verification ==="
+_fail=0
+check() { # check