perf: native ixformer decode (v1 ≤32K, v2 >32K) + flash_attn_varlen prefill
Replaces all Python PyTorch fallback attention with native ixformer kernels: Decode path: - ≤32K: paged_attention_v1 (5D KV layout, x=8) — verified on real BI-V100 - >32K: paged_attention_v2 (5D→4D permute) — verified 65K+ on real BI-V100 - Removes _forward_decode_pytorch Python fallback entirely Prefill path (profiling): - _run_sdpa_fallback now uses ixformer.flash_attn_varlen_func - head_dim=256 verified correct (diff<0.004) and 1.7x faster than PyTorch - Falls back to Q-tiling pure-math if ixformer unavailable Also includes: MoE kernel integration, GDN C++ kernels, diagnostic scripts, xllm upstream layer/kernel references, .dockerignore cleanup. All changes verified on real BI-V100 hardware (single card).
This commit is contained in:
28
probe_paged_attn.py
Normal file
28
probe_paged_attn.py
Normal file
@@ -0,0 +1,28 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Probe ixformer.vllm_single_query_cached_kv_attention signature and test."""
|
||||
import inspect
|
||||
import torch
|
||||
import ixformer
|
||||
|
||||
# Print signature
|
||||
fn = ixformer.vllm_single_query_cached_kv_attention
|
||||
print(f"Signature: {inspect.signature(fn)}")
|
||||
|
||||
# Also check v2
|
||||
if hasattr(ixformer, 'vllm_single_query_cached_kv_attention_v2'):
|
||||
fn2 = ixformer.vllm_single_query_cached_kv_attention_v2
|
||||
print(f"V2 Signature: {inspect.signature(fn2)}")
|
||||
|
||||
# Check contrib.vllm_flash_attn if available
|
||||
try:
|
||||
from ixformer.contrib import vllm_flash_attn
|
||||
print(f"\nvllm_flash_attn dir: {[x for x in dir(vllm_flash_attn) if not x.startswith('_')]}")
|
||||
except Exception as e:
|
||||
print(f"\nvllm_flash_attn: {e}")
|
||||
|
||||
# Check ixformer.vllm submodule
|
||||
try:
|
||||
import ixformer.vllm as ixv
|
||||
print(f"\nixformer.vllm dir: {[x for x in dir(ixv) if not x.startswith('_')]}")
|
||||
except Exception as e:
|
||||
print(f"\nixformer.vllm: {e}")
|
||||
Reference in New Issue
Block a user