Files
enginex-ascend-910-vllm/tests/e2e/generate_coverage_md.py
Sun Ruoxi 7f8a1b1f7a init v0.23.0
Signed-off-by: Sun Ruoxi <sunruoxi@4paradigm.com>
2026-08-27 15:11:51 +08:00

1328 lines
49 KiB
Python

#!/usr/bin/env python3
"""Generate coverage.md by scanning test files (Python + YAML) and
extracting feature flags via AST and YAML parsing."""
import ast
import contextlib
import json
from pathlib import Path
from typing import Any
import regex as re
REPO_ROOT = Path(__file__).resolve().parents[2]
E2E_PR_ROOT = REPO_ROOT / "tests" / "e2e" / "pull_request"
OUTPUT_FILE = Path(__file__).resolve().parent / "coverage.md"
COLUMNS = [
"Test file",
"Test method",
"Model",
"310P",
"Dense",
"MoE",
"Embedding",
"Classification",
"Reranker",
"Mamba/SSM",
"Multimodal Reasoning",
"TP",
"PP",
"EP",
"PCP",
"DCP",
"Context Parallel",
"EPLB",
"Dynamic EPLB",
"Multistream MoE",
"Full Graph",
"Full Decode Only Graph",
"Default FULL_AND_PIECEWISE Graph",
"Piecewise Graph",
"Eager Mode",
"PD disaggregation",
"W8A8",
"W4A8",
"FP16",
"LoRA",
"Multi-LoRA",
"Runtime LoRA updating",
"Fully sharded LoRA parameterization",
"Spec Decode",
"MTP",
"Eagle-3",
"SFA/DSA",
"DSA CP",
"Pooling runner",
"Score API",
"Classification API",
"Distributed executor mp",
"Flash Attention 3",
"FIA comparison",
"Chunked Prefill",
"Prefix Caching",
"CPU/KV offloading",
"KV transfer/events",
"Sleep/Wake memory",
"Xlite Graph",
"CP KV Interleave",
"Long Sequence",
"FlashComm1 env",
"Skipped",
"Conditional skip",
"Logprobs",
"Batch inference",
"Mixed lengths",
]
CARD_SECTIONS = [
("1-Card Tests", "one_card"),
("2-Card Tests", "two_card"),
("4-Card Tests", "four_card"),
]
CHECK = "\u2705"
EMPTY = ""
def _source_to_str(node):
if isinstance(node, ast.Constant):
return str(node.value)
if isinstance(node, ast.Str):
return node.s
return None
def _flatten_dict_keys(d):
out = set()
if isinstance(d, dict):
for k, v in d.items():
out.add(k)
if isinstance(v, dict):
out.update(_flatten_dict_keys(v))
return out
def _extract_kwarg_dict(call_node, key):
for kw in call_node.keywords:
if kw.arg == key and isinstance(kw.value, ast.Dict):
d = {}
for dk, dv in zip(kw.value.keys, kw.value.values):
ks = _source_to_str(dk)
vs = _source_to_str(dv)
if ks is not None:
d[ks] = vs
elif isinstance(dk, ast.Constant):
ks2 = str(dk.value)
d[ks2] = vs
return d
return None
def _extract_kwarg_value(call_node, key):
for kw in call_node.keywords:
if kw.arg == key:
return _source_to_str(kw.value)
return None
def _extract_kwarg_int(call_node, key):
for kw in call_node.keywords:
if kw.arg == key and isinstance(kw.value, ast.Constant):
try:
return int(kw.value.value)
except (TypeError, ValueError):
return None
return None
def _extract_kwarg_bool(call_node, key):
for kw in call_node.keywords:
if kw.arg == key and isinstance(kw.value, ast.Constant):
return bool(kw.value.value)
return None
def _find_calls_in_body(body, func_name):
calls = []
for node in _walk_body(body):
if isinstance(node, ast.Call):
if (
isinstance(node.func, ast.Name)
and node.func.id == func_name
or isinstance(node.func, ast.Attribute)
and node.func.attr == func_name
):
calls.append(node)
return calls
def _find_all_calls_in_body(body):
calls = []
for node in _walk_body(body):
if isinstance(node, ast.Call):
calls.append(node)
return calls
def _extract_model_from_call(call_node):
model = _extract_kwarg_value(call_node, "model_name") or _extract_kwarg_value(call_node, "model")
if model:
return model
pos_args = call_node.args
if pos_args and isinstance(pos_args[0], ast.Constant):
return str(pos_args[0].value)
return None
def _extract_config_from_call(call_node):
config_keys = set()
for kw in call_node.keywords:
config_keys.add(kw.arg)
comp_config = _extract_kwarg_dict(call_node, "compilation_config")
if comp_config:
config_keys.update(_flatten_dict_keys(comp_config))
spec_config = _extract_kwarg_dict(call_node, "speculative_config")
if spec_config:
config_keys.update(_flatten_dict_keys(spec_config))
add_config = _extract_kwarg_dict(call_node, "additional_config")
if add_config:
config_keys.update(_flatten_dict_keys(add_config))
return config_keys
def _detect_cards(rel_path):
parts = rel_path.split("/")
for p in parts:
if p == "one_card":
return 1
if p == "two_card":
return 2
if p == "four_card":
return 4
return 1
def _detect_310p(rel_path):
return "_310p" in rel_path or "/310p/" in rel_path or rel_path.startswith("310p/")
def _classify_model(model_name):
flags = {
"Dense": False,
"MoE": False,
"Embedding": False,
"Classification": False,
"Reranker": False,
"Mamba/SSM": False,
"Multimodal Reasoning": False,
}
if not model_name:
return flags
ml = model_name.lower()
is_moe = any(
x in model_name
for x in [
"A3B",
"MoE",
"DeepSeek-V2",
"DeepSeek-V3",
"DeepSeek-V4",
"Qwen3-30B-A3B",
"Qwen3.5-35B-A3B",
"Qwen3-Next-80B-A3B",
"Qwen3-Coder-30B-A3B",
]
)
is_embedding = any(x in ml for x in ["embedding", "e5", "bge-m3", "minilm"]) or any(
x in model_name for x in ["Embedding"]
)
is_classification = any(x in ml for x in ["apeach", "classification"]) or "SequenceClassification" in model_name
is_reranker = any(x in ml for x in ["reranker"])
is_mamba_ssm = "Qwen3.5" in model_name and "VL" not in model_name
is_vl = any(x in model_name for x in ["VL", "HunyuanOCR", "Audio", "whisper", "MiniCPM-V"])
if is_vl:
flags["Multimodal Reasoning"] = True
if is_mamba_ssm and not is_vl:
flags["Mamba/SSM"] = True
if is_reranker:
flags["Reranker"] = True
if is_classification:
flags["Classification"] = True
if is_embedding:
flags["Embedding"] = True
if is_moe:
flags["MoE"] = True
is_other = not (is_moe or is_embedding or is_classification or is_reranker or is_vl)
if is_other and not is_mamba_ssm:
flags["Dense"] = True
return flags
def _extract_env_vars(body):
env_vars = {}
for node in _walk_body(body):
if isinstance(node, ast.Call):
fn = None
if isinstance(node.func, ast.Name):
fn = node.func.id
elif isinstance(node.func, ast.Attribute):
fn = node.func.attr
if fn == "setenv" and node.args:
key = _source_to_str(node.args[0])
val = _source_to_str(node.args[1]) if len(node.args) > 1 else None
if key:
env_vars[key] = val
if fn == "patch" and isinstance(node.func, ast.Attribute):
pass
for node in _walk_body(body):
if isinstance(node, ast.Subscript):
try:
if (
(isinstance(node.value, ast.Attribute) and node.value.attr == "environ")
or isinstance(node.value, ast.Name)
and node.value.id == "environ"
):
sl = node.slice
if isinstance(sl, ast.Constant):
env_vars[str(sl.value)] = None
except Exception:
pass
for node in _walk_body(body):
if isinstance(node, ast.Call) and isinstance(node.func, ast.Attribute):
if node.func.attr == "dict" and isinstance(node.func.value, ast.Name):
if node.func.value.id == "patch":
for kw in node.keywords:
if kw.arg == "os.environ" and isinstance(kw.value, ast.Dict):
for dk, dv in zip(kw.value.keys, kw.value.values):
ks = _source_to_str(dk)
vs = _source_to_str(dv)
if ks:
env_vars[ks] = vs
return env_vars
def _extract_decorators(func_node):
has_skip = False
has_skipif = False
for dec in func_node.decorator_list:
if isinstance(dec, ast.Call) and isinstance(dec.func, ast.Name):
if dec.func.id == "pytest.mark.skip":
has_skip = True
if dec.func.id == "pytest.mark.skipif":
has_skipif = True
if isinstance(dec, ast.Call) and isinstance(dec.func, ast.Attribute):
if dec.func.attr == "skip":
has_skip = True
if dec.func.attr == "skipif":
has_skipif = True
if isinstance(dec, ast.Name) and dec.id == "pytest.mark.skip":
has_skip = True
return has_skip, has_skipif
def _walk_body(body):
if isinstance(body, list):
for stmt in body:
yield from ast.walk(stmt)
else:
yield from ast.walk(body)
def _find_llm_or_vllmrunner_calls(body):
calls = []
for node in _walk_body(body):
if isinstance(node, ast.Call):
if (
isinstance(node.func, ast.Name)
and node.func.id in ("LLM", "VllmRunner")
or isinstance(node.func, ast.Name)
and node.func.id == "compare_logprobs"
or isinstance(node.func, ast.Name)
and node.func.id == "check_outputs_equal"
):
calls.append(node)
return calls
def _extract_all_kwarg_keys(body):
keys = set()
for node in _walk_body(body):
if isinstance(node, ast.Call):
for kw in node.keywords:
if kw.arg:
keys.add(kw.arg)
return keys
def _find_method_calls(body):
method_names = set()
for node in _walk_body(body):
if isinstance(node, ast.Call) and isinstance(node.func, ast.Attribute):
method_names.add(node.func.attr)
return method_names
def _find_lora_requests(body):
count = 0
for node in _walk_body(body):
if isinstance(node, ast.Call):
if isinstance(node.func, ast.Name) and node.func.id == "LoRARequest":
count += 1
return count
def _extract_models_from_parametrize(func_node, source_code):
models = []
for dec in func_node.decorator_list:
if isinstance(dec, ast.Call) and isinstance(dec.func, ast.Attribute):
if dec.func.attr == "parametrize":
for kw in dec.keywords:
if kw.arg in ("model", "model_name", "model_setup") and isinstance(kw.value, ast.List):
for elt in kw.value.elts:
if isinstance(elt, ast.Constant):
val = elt.value
if isinstance(val, str):
models.append(val)
elif isinstance(val, tuple):
models.append(val[1] if len(val) > 1 else str(val))
elif kw.arg == "method" and isinstance(kw.value, ast.Dict):
pass
return models
def _extract_server_args_models(body, source_code):
models = set()
for node in _walk_body(body):
if isinstance(node, ast.Call):
if isinstance(node.func, ast.Name) and node.func.id in (
"RemoteOpenAIServer",
"RemotePDServer",
"RemoteEPDServer",
):
for arg in node.args:
if isinstance(arg, ast.Constant):
models.add(str(arg.value))
for kw in node.keywords:
if kw.arg == "model" and isinstance(kw.value, ast.Constant):
models.add(str(kw.value.value))
if not models:
docstring_val = None
if isinstance(body, list) and body and isinstance(body[0], ast.Expr):
if isinstance(body[0].value, ast.Constant) and isinstance(body[0].value.value, str):
docstring_val = body[0].value.value
str_literals = set()
for node in _walk_body(body):
if isinstance(node, ast.Constant) and isinstance(node.value, str):
v = node.value
if v == docstring_val:
continue
if len(v) > 200:
continue
if any(
p in v
for p in [
"Qwen",
"DeepSeek",
"MiniCPM",
"ilama",
"LLM-Research",
"gdydems",
"BAAI",
"intfloat",
"sentence-transformers",
"Howeee",
"vllm-ascend",
"charent",
"RedHatAI",
"z-lab",
"amd",
"MNN",
"wemaster",
"dengcao",
"openai-mirror",
"openbmb",
"OpenBMB",
"Meta-Llama",
"amazon",
]
):
if not v.startswith("--") and "/" in v and len(v) > 5:
str_literals.add(v)
models = str_literals
return models
def _process_test_file(filepath, source_code, root_path=None):
try:
tree = ast.parse(source_code)
except SyntaxError:
return []
if root_path is None:
root_path = E2E_PR_ROOT
rel_path = str(filepath.relative_to(root_path))
is_310p = _detect_310p(rel_path)
card_prefix = ""
for prefix in ("one_card/", "two_card/", "four_card/"):
if rel_path.startswith(prefix):
card_prefix = prefix
break
display_path = rel_path[len(card_prefix) :] if card_prefix else rel_path
results = []
for node in ast.walk(tree):
if not isinstance(node, ast.FunctionDef):
continue
if not node.name.startswith("test_"):
continue
test_name = node.name
has_skip, has_skipif = _extract_decorators(node)
all_calls = _find_llm_or_vllmrunner_calls(node.body)
config_keys = _extract_all_kwarg_keys(node.body)
kwarg_vals = {}
for call in all_calls:
for kw in call.keywords:
if kw.arg and isinstance(kw.value, ast.Constant):
kwarg_vals[kw.arg] = kw.value.value
env_vars = _extract_env_vars(node.body)
method_calls = _find_method_calls(node.body)
lora_count = _find_lora_requests(node.body)
models_from_params = _extract_models_from_parametrize(node, source_code)
server_models = _extract_server_args_models(node.body, source_code)
all_models = []
if models_from_params:
all_models = list(models_from_params)
elif server_models:
all_models = sorted(server_models)
else:
found_models = set()
for call in all_calls:
m = _extract_model_from_call(call)
if m:
found_models.add(m)
for call in all_calls:
if isinstance(call.func, ast.Name) and call.func.id == "compare_logprobs":
m = _extract_kwarg_value(call, "model_name")
if m:
found_models.add(m)
for var_name in ("model", "model_name", "MODEL_NAME", "DEFAULT_MODEL", "MODEL_PATH"):
for assign in _walk_body(node.body):
if isinstance(assign, ast.Assign):
for target in assign.targets:
if isinstance(target, ast.Name) and target.id == var_name:
val = _source_to_str(assign.value)
if val and "/" in val:
found_models.add(val)
for var_name in ("MODELS", "CROSS_ENCODER_MODELS", "EMBEDDING_MODELS", "WHISPER_MODELS", "MINICPM_MODELS"):
for assign in ast.walk(tree):
if isinstance(assign, ast.Assign):
for target in assign.targets:
if isinstance(target, ast.Name) and target.id == var_name:
if isinstance(assign.value, ast.List):
for elt in assign.value.elts:
v = _source_to_str(elt)
if v and "/" in v:
found_models.add(v)
if found_models:
all_models = sorted(found_models)
if not all_models:
model_str = "-"
else:
model_str = "<br>".join(all_models)
model_flags = {}
for m in all_models:
mf = _classify_model(m)
for k, v in mf.items():
if v:
model_flags[k] = True
if not model_flags and not all_models:
model_flags = {}
tp = _extract_kwarg_int_from_all(all_calls, "tensor_parallel_size")
pp = _extract_kwarg_int_from_all(all_calls, "pipeline_parallel_size")
ep = _extract_kwarg_bool_from_all(all_calls, "enable_expert_parallel")
pcp = _extract_kwarg_int_from_all(all_calls, "prefill_context_parallel_size")
dcp = _extract_kwarg_int_from_all(all_calls, "decode_context_parallel_size")
enforce_eager = _extract_kwarg_bool_from_all(all_calls, "enforce_eager")
dist_exec_mp = _extract_kwarg_bool_from_all(all_calls, "distributed_executor_backend")
cudagraph_mode = _extract_cudagraph_mode(all_calls)
has_cudagraph_sizes = _has_cudagraph_capture_sizes(all_calls)
enable_lora = _extract_kwarg_bool_from_all(all_calls, "enable_lora")
max_loras = _extract_kwarg_int_from_all(all_calls, "max_loras")
fully_sharded = _extract_kwarg_bool_from_all(all_calls, "fully_sharded_loras")
spec_config = None
for call in all_calls:
sc = _extract_kwarg_dict(call, "speculative_config")
if sc:
spec_config = sc
kv_transfer_config = None
for call in all_calls:
kc = _extract_kwarg_dict(call, "kv_transfer_config")
if kc:
kv_transfer_config = kc
kv_events_config = None
for call in all_calls:
kec = _extract_kwarg_dict(call, "kv_events_config")
if kec:
kv_events_config = kec
runner = _extract_kwarg_value_from_all(all_calls, "runner")
enable_prefix_caching = _extract_kwarg_bool_from_all(all_calls, "enable_prefix_caching")
enable_chunked_prefill = _extract_kwarg_bool_from_all(all_calls, "enable_chunked_prefill")
cp_kv_interleave = _extract_kwarg_int_from_all(all_calls, "cp_kv_cache_interleave_size")
max_model_len = _extract_kwarg_int_from_all(all_calls, "max_model_len")
limit_mm = _extract_kwarg_value_from_all(all_calls, "limit_mm_per_prompt")
attention_backend = _extract_kwarg_value_from_all(all_calls, "attention_backend")
quantization = _extract_kwarg_value_from_all(all_calls, "quantization")
dtype = _extract_kwarg_value_from_all(all_calls, "dtype")
sleep_mode = _extract_kwarg_bool_from_all(all_calls, "enable_sleep_mode")
add_config = None
for call in all_calls:
ac = _extract_kwarg_dict(call, "additional_config")
if ac:
add_config = ac
for call in all_calls:
cc = _extract_kwarg_dict(call, "compilation_config")
if cc:
break
xlite_graph = None
for call in all_calls:
xc = _extract_kwarg_dict(call, "additional_config")
if xc and "xlite_graph_config" in xc:
xlite_graph = xc["xlite_graph_config"]
enable_dsa_cp = None
multistream_moe = None
enable_flashcomm1 = None
if add_config:
if "enable_dsa_cp" in add_config:
enable_dsa_cp = add_config["enable_dsa_cp"]
if "enable_multistream_moe" in add_config:
multistream_moe = add_config["enable_multistream_moe"]
if "multistream_overlap_shared_expert" in add_config:
multistream_moe = add_config["multistream_overlap_shared_expert"]
if "enable_flashcomm1" in add_config:
enable_flashcomm1 = add_config["enable_flashcomm1"]
eplb_config = None
for call in all_calls:
ec = _extract_kwarg_dict(call, "additional_config")
if ec and "eplb_config" in ec:
eplb_config = ec["eplb_config"]
has_mamba_ssm_dtype = "mamba_ssm_cache_dtype" in config_keys
row = {}
row["Test file"] = display_path
row["_orig_rel_path"] = rel_path
row["Test method"] = test_name
row["Model"] = model_str
row["310P"] = CHECK if is_310p else EMPTY
row["Dense"] = CHECK if model_flags.get("Dense") else EMPTY
row["MoE"] = CHECK if model_flags.get("MoE") else EMPTY
row["Embedding"] = CHECK if model_flags.get("Embedding") else EMPTY
row["Classification"] = CHECK if model_flags.get("Classification") else EMPTY
row["Reranker"] = CHECK if model_flags.get("Reranker") else EMPTY
row["Mamba/SSM"] = CHECK if (model_flags.get("Mamba/SSM") or has_mamba_ssm_dtype) else EMPTY
row["Multimodal Reasoning"] = CHECK if (model_flags.get("Multimodal Reasoning") or limit_mm) else EMPTY
row["TP"] = CHECK if (tp is not None and tp > 1) else EMPTY
row["PP"] = CHECK if (pp is not None and pp > 1) else EMPTY
row["EP"] = CHECK if ep else EMPTY
row["PCP"] = CHECK if (pcp is not None and pcp > 1) else EMPTY
row["DCP"] = CHECK if (dcp is not None and dcp > 1) else EMPTY
row["Context Parallel"] = CHECK if ((pcp is not None and pcp > 1) or (dcp is not None and dcp > 1)) else EMPTY
has_eplb = (
eplb_config is not None or "expert_parallel_load_balancing" in config_keys or "eplb_config" in config_keys
)
row["EPLB"] = CHECK if has_eplb else EMPTY
has_dynamic_eplb = False
if eplb_config and isinstance(eplb_config, dict):
has_dynamic_eplb = eplb_config.get("dynamic_eplb") or "eplb_policy_type" in eplb_config
if "dynamic_eplb" in config_keys or "DYNAMIC_EPLB" in env_vars:
has_dynamic_eplb = True
row["Dynamic EPLB"] = CHECK if has_dynamic_eplb else EMPTY
row["Multistream MoE"] = CHECK if multistream_moe else EMPTY
if cudagraph_mode == "FULL":
row["Full Graph"] = CHECK
elif cudagraph_mode == "FULL_DECODE_ONLY":
row["Full Decode Only Graph"] = CHECK
elif cudagraph_mode == "PIECEWISE":
row["Piecewise Graph"] = CHECK
elif cudagraph_mode == "default":
if enforce_eager:
row["Eager Mode"] = CHECK
elif has_cudagraph_sizes:
row["Default FULL_AND_PIECEWISE Graph"] = CHECK
else:
row["Eager Mode"] = CHECK
elif enforce_eager:
row["Eager Mode"] = CHECK
else:
row["Eager Mode"] = EMPTY
has_pd_disagg = False
if kv_transfer_config:
kv_role = kv_transfer_config.get("kv_role")
if kv_role and ("kv_producer" in str(kv_role) or "kv_consumer" in str(kv_role)):
has_pd_disagg = True
if "pd_disaggregation" in config_keys:
has_pd_disagg = True
for node2 in _walk_body(node.body):
if isinstance(node2, ast.Name) and node2.id in ("RemotePDServer", "DisaggPDProxy"):
has_pd_disagg = True
row["PD disaggregation"] = CHECK if has_pd_disagg else EMPTY
is_w8a8 = any("w8a8" in m.lower() for m in all_models) or (
quantization == "ascend" and any("w8a8" in m.lower() for m in all_models)
)
is_w4a8 = any("w4a8" in m.lower() for m in all_models) or (
quantization == "ascend" and any("w4a8" in m.lower() for m in all_models)
)
row["W8A8"] = CHECK if is_w8a8 else EMPTY
row["W4A8"] = CHECK if is_w4a8 else EMPTY
is_fp16 = dtype in ("float16", "half") or (is_310p and dtype != "bfloat16" and not is_w8a8)
row["FP16"] = CHECK if is_fp16 else EMPTY
row["LoRA"] = CHECK if enable_lora else EMPTY
row["Multi-LoRA"] = (
CHECK if (lora_count > 2 or (max_loras is not None and max_loras > 1 and lora_count >= 2)) else EMPTY
)
has_runtime_lora = False
for mn in method_calls:
if mn in ("add_lora", "remove_lora", "reload_lora"):
has_runtime_lora = True
if "VLLM_ALLOW_RUNTIME_LORA_UPDATING" in env_vars:
has_runtime_lora = True
row["Runtime LoRA updating"] = CHECK if has_runtime_lora else EMPTY
row["Fully sharded LoRA parameterization"] = CHECK if fully_sharded else EMPTY
has_spec = spec_config is not None or "speculative_config" in config_keys or "speculative_method" in config_keys
row["Spec Decode"] = CHECK if has_spec else EMPTY
has_mtp = False
if spec_config and isinstance(spec_config, dict):
method = spec_config.get("method", "")
if method and "mtp" in method.lower():
has_mtp = True
if method in ("deepseek_mtp", "qwen3_5_mtp", "qwen3_next_mtp"):
has_mtp = True
if "mtp" in test_name.lower():
has_mtp = True
row["MTP"] = CHECK if has_mtp else EMPTY
has_eagle3 = False
if spec_config and isinstance(spec_config, dict):
method = spec_config.get("method", "")
if method and ("eagle3" in method.lower() or method == "eagle"):
spec_model = spec_config.get("model") or ""
if "eagle3" in spec_model.lower() or method == "eagle3":
has_eagle3 = True
for m in all_models:
if "eagle3" in m.lower():
has_eagle3 = True
row["Eagle-3"] = CHECK if has_eagle3 else EMPTY
has_sfa_dsa = False
if (
enable_dsa_cp
or "enable_dsa_cp" in config_keys
or "sfa" in str(add_config).lower()
or "dsa" in str(add_config).lower()
):
has_sfa_dsa = True
if has_sfa_dsa:
row["SFA/DSA"] = CHECK
else:
row["SFA/DSA"] = EMPTY
has_dsa_cp = enable_dsa_cp and ((pcp is not None and pcp > 1) or (dcp is not None and dcp > 1))
row["DSA CP"] = CHECK if has_dsa_cp else EMPTY
row["Pooling runner"] = (
CHECK if (runner == "pooling" or "task" in config_keys and kwarg_vals.get("task") == "pooling") else EMPTY
)
row["Score API"] = CHECK if "score" in method_calls else EMPTY
row["Classification API"] = CHECK if "classify" in method_calls else EMPTY
row["Distributed executor mp"] = CHECK if dist_exec_mp else EMPTY
has_fa3 = False
if attention_backend and "FLASH_ATTN" in str(attention_backend):
has_fa3 = True
if "FA3" in str(attention_backend):
has_fa3 = True
if "FLASH_ATTN" in str(attention_backend):
has_fa3 = True
row["Flash Attention 3"] = CHECK if has_fa3 else EMPTY
has_fia = False
if "FIA" in test_name or "fia" in source_code.lower()[:5000]:
has_fia = True
for m in method_calls:
if "_assert_outputs_match" in m or "_generate_with_backend" in m:
has_fia = True
has_fa3_comparison = has_fa3 and has_fia
row["FIA comparison"] = CHECK if has_fa3_comparison else EMPTY
row["Chunked Prefill"] = CHECK if (enable_chunked_prefill or "enable_chunked_prefill" in config_keys) else EMPTY
row["Prefix Caching"] = CHECK if (enable_prefix_caching or "enable_prefix_caching" in config_keys) else EMPTY
has_cpu_offloading = False
if "OffloadingConnector" in source_code or "cpu_offloading" in config_keys or "kv_connector" in config_keys:
for call in all_calls:
kc = _extract_kwarg_dict(call, "kv_transfer_config")
if kc and kc.get("kv_connector") == "OffloadingConnector":
has_cpu_offloading = True
row["CPU/KV offloading"] = CHECK if has_cpu_offloading else EMPTY
has_kv_transfer = False
if kv_transfer_config:
connector = kv_transfer_config.get("kv_connector")
if connector and connector != "OffloadingConnector" and connector != "ExampleHiddenStatesConnector":
has_kv_transfer = True
if kv_events_config:
has_kv_transfer = True
row["KV transfer/events"] = CHECK if has_kv_transfer else EMPTY
row["Sleep/Wake memory"] = CHECK if sleep_mode else EMPTY
row["Xlite Graph"] = CHECK if xlite_graph else EMPTY
row["CP KV Interleave"] = CHECK if (cp_kv_interleave is not None and cp_kv_interleave > 0) else EMPTY
is_context_parallel = (
(max_model_len is not None and max_model_len > 8192)
or "context_parallel" in rel_path
or "context_parallel" in test_name
)
row["Context Parallel"] = CHECK if is_context_parallel else EMPTY
has_flashcomm1 = (
"VLLM_ASCEND_ENABLE_FLASHCOMM1" in env_vars and env_vars.get("VLLM_ASCEND_ENABLE_FLASHCOMM1") == "1"
)
if enable_flashcomm1:
has_flashcomm1 = True
row["FlashComm1 env"] = CHECK if has_flashcomm1 else EMPTY
row["Skipped"] = CHECK if has_skip else EMPTY
row["Conditional skip"] = CHECK if has_skipif else EMPTY
has_logprobs = "logprobs" in config_keys or "prompt_logprobs" in config_keys or "num_logprobs" in config_keys
if "logprobs" in method_calls or "generate_greedy_logprobs" in method_calls:
has_logprobs = True
if "compare_logprobs" in method_calls:
has_logprobs = True
row["Logprobs"] = CHECK if has_logprobs else EMPTY
has_batch = False
for node2 in _walk_body(node.body):
if isinstance(node2, ast.List):
if len(node2.elts) > 1:
for elt in node2.elts:
if isinstance(elt, ast.Constant) and isinstance(elt.value, str):
if len(elt.value) > 5:
has_batch = True
if "max_num_seqs" in config_keys:
mn_seqs = kwarg_vals.get("max_num_seqs")
if mn_seqs and mn_seqs > 1:
has_batch = True
if "batch" in test_name.lower():
has_batch = True
row["Batch inference"] = CHECK if has_batch else EMPTY
has_mixed_lengths = (
"mixed" in test_name.lower() or "mixed_length" in config_keys or "mixed_lengths" in test_name.lower()
)
for node2 in _walk_body(node.body):
if isinstance(node2, ast.Constant) and isinstance(node2.value, str):
if "mixed" in node2.value.lower() and "length" in node2.value.lower():
has_mixed_lengths = True
row["Mixed lengths"] = CHECK if has_mixed_lengths else EMPTY
results.append(row)
return results
def _extract_kwarg_int_from_all(calls, key):
for call in calls:
v = _extract_kwarg_int(call, key)
if v is not None:
return v
return None
def _extract_kwarg_bool_from_all(calls, key):
for call in calls:
v = _extract_kwarg_bool(call, key)
if v is not None:
return v
return None
def _extract_kwarg_value_from_all(calls, key):
for call in calls:
v = _extract_kwarg_value(call, key)
if v is not None:
return v
return None
def _extract_cudagraph_mode(all_calls):
for call in all_calls:
cc = _extract_kwarg_dict(call, "compilation_config")
if cc:
mode = cc.get("cudagraph_mode")
if mode:
cudagraph_mode_map = {
"FULL_DECODE_ONLY": "FULL_DECODE_ONLY",
"PIECEWISE": "PIECEWISE",
"FULL": "FULL",
}
return cudagraph_mode_map[mode]
cg_mode = _extract_kwarg_value(call, "cudagraph_mode")
if cg_mode:
return cg_mode
return "default"
def _has_cudagraph_capture_sizes(all_calls):
for call in all_calls:
cc = _extract_kwarg_dict(call, "compilation_config")
if cc and "cudagraph_capture_sizes" in cc:
return True
if _extract_kwarg_value(call, "cudagraph_capture_sizes") is not None:
return True
return False
def _parse_server_cmd(server_cmd):
features: dict[str, Any] = {}
if isinstance(server_cmd, str):
args = server_cmd.split()
elif isinstance(server_cmd, list):
args = [str(a) for a in server_cmd]
else:
return features
def _try_json(raw):
if isinstance(raw, str):
raw = raw.strip("'").strip('"')
try:
return json.loads(raw)
except (json.JSONDecodeError, ValueError):
pass
return None
JSON_FLAGS = {
"--compilation-config": "compilation_config",
"--speculative-config": "speculative_config",
"--kv-transfer-config": "kv_transfer_config",
"--additional-config": "additional_config",
}
i = 0
while i < len(args):
a = args[i]
if a == "--tensor-parallel-size" and i + 1 < len(args):
with contextlib.suppress(ValueError):
features["tp"] = int(args[i + 1])
i += 2
elif a == "--pipeline-parallel-size" and i + 1 < len(args):
with contextlib.suppress(ValueError):
features["pp"] = int(args[i + 1])
i += 2
elif a == "--enforce-eager":
features["enforce_eager"] = True
i += 1
elif a == "--enable-expert-parallel":
features["ep"] = True
i += 1
elif a in ("--quantization", "--dtype", "--attention-backend", "--runner") and i + 1 < len(args):
features[a.removeprefix("--").replace("-", "_")] = args[i + 1]
i += 2
elif a in JSON_FLAGS and i + 1 < len(args):
val = _try_json(args[i + 1])
if val is not None:
features[JSON_FLAGS[a]] = val
i += 2
elif a == "--enable-prefix-caching":
features["enable_prefix_caching"] = True
i += 1
elif a == "--no-enable-prefix-caching":
features["enable_prefix_caching"] = False
i += 1
elif a == "--enable-chunked-prefill":
features["enable_chunked_prefill"] = True
i += 1
elif a == "--max-model-len" and i + 1 < len(args):
with contextlib.suppress(ValueError):
features["max_model_len"] = int(args[i + 1])
i += 2
elif a == "--cp-kv-cache-interleave-size" and i + 1 < len(args):
with contextlib.suppress(ValueError):
features["cp_kv_interleave"] = int(args[i + 1])
i += 2
elif a == "--enable-sleep-mode":
features["enable_sleep_mode"] = True
i += 1
elif a == "--enable-lora":
features["enable_lora"] = True
i += 1
elif a == "--max-loras" and i + 1 < len(args):
with contextlib.suppress(ValueError):
features["max_loras"] = int(args[i + 1])
pass
i += 2
elif a == "--fully-sharded-loras":
features["fully_sharded_loras"] = True
i += 1
elif a == "--distributed-executor-backend" and i + 1 < len(args):
features["dist_exec_mp"] = True
i += 2
else:
i += 1
if isinstance(server_cmd, str):
for flag, key in JSON_FLAGS.items():
if key not in features:
m = re.search(rf"{flag}\s+'([^']+)'", server_cmd)
if m:
val = _try_json(m.group(1))
if val is not None:
features[key] = val
return features
def _process_yaml_file(filepath, source_code, root_path):
try:
import yaml
data = yaml.safe_load(source_code)
except Exception:
return []
if not isinstance(data, dict):
return []
rel_path = str(filepath.relative_to(root_path))
is_310p = _detect_310p(rel_path)
display_path = rel_path
def _build_row(test_name, model, features):
model_flags = _classify_model(model) if model else {}
tp = features.get("tp")
pp = features.get("pp")
ep = features.get("ep")
enforce_eager = features.get("enforce_eager")
dtype = features.get("dtype")
compilation_config = features.get("compilation_config")
speculative_config = features.get("speculative_config")
kv_transfer_config = features.get("kv_transfer_config")
additional_config = features.get("additional_config")
enable_prefix_caching = features.get("enable_prefix_caching")
enable_chunked_prefill = features.get("enable_chunked_prefill")
cp_kv_interleave = features.get("cp_kv_interleave")
max_model_len = features.get("max_model_len")
enable_lora = features.get("enable_lora")
max_loras = features.get("max_loras")
fully_sharded = features.get("fully_sharded_loras")
attention_backend = features.get("attention_backend")
runner = features.get("runner")
sleep_mode = features.get("enable_sleep_mode")
dist_exec_mp = features.get("dist_exec_mp")
env_vars = features.get("envs", {})
cudagraph_mode = "default"
has_cudagraph_sizes = False
if compilation_config:
mode = compilation_config.get("cudagraph_mode", "")
if mode:
cudagraph_mode = mode
if "cudagraph_capture_sizes" in compilation_config:
has_cudagraph_sizes = True
eplb_config = None
multistream_moe = None
enable_dsa_cp = None
enable_flashcomm1 = None
if additional_config:
if "eplb_config" in additional_config:
eplb_config = additional_config["eplb_config"]
if "enable_multistream_moe" in additional_config:
multistream_moe = additional_config["enable_multistream_moe"]
if "multistream_overlap_shared_expert" in additional_config:
multistream_moe = additional_config["multistream_overlap_shared_expert"]
if "enable_dsa_cp" in additional_config:
enable_dsa_cp = additional_config["enable_dsa_cp"]
if "enable_flashcomm1" in additional_config:
enable_flashcomm1 = additional_config["enable_flashcomm1"]
if "ascend_compilation_config" in additional_config:
ascend_cc = additional_config["ascend_compilation_config"]
if ascend_cc.get("enable_npugraph_ex"):
has_cudagraph_sizes = True
row = {}
row["Test file"] = display_path
row["_orig_rel_path"] = rel_path
row["Test method"] = test_name
row["Model"] = model if model else "-"
row["310P"] = CHECK if is_310p else EMPTY
row["Dense"] = CHECK if model_flags.get("Dense") else EMPTY
row["MoE"] = CHECK if model_flags.get("MoE") else EMPTY
row["Embedding"] = CHECK if model_flags.get("Embedding") else EMPTY
row["Classification"] = CHECK if model_flags.get("Classification") else EMPTY
row["Reranker"] = CHECK if model_flags.get("Reranker") else EMPTY
row["Mamba/SSM"] = CHECK if model_flags.get("Mamba/SSM") else EMPTY
row["Multimodal Reasoning"] = CHECK if model_flags.get("Multimodal Reasoning") else EMPTY
row["TP"] = CHECK if (tp is not None and tp > 1) else EMPTY
row["PP"] = CHECK if (pp is not None and pp > 1) else EMPTY
row["EP"] = CHECK if ep else EMPTY
row["PCP"] = EMPTY
row["DCP"] = EMPTY
row["Context Parallel"] = EMPTY
row["EPLB"] = CHECK if eplb_config else EMPTY
row["Dynamic EPLB"] = EMPTY
row["Multistream MoE"] = CHECK if multistream_moe else EMPTY
if cudagraph_mode == "FULL":
row["Full Graph"] = CHECK
elif cudagraph_mode == "FULL_DECODE_ONLY":
row["Full Decode Only Graph"] = CHECK
elif cudagraph_mode == "PIECEWISE":
row["Piecewise Graph"] = CHECK
elif cudagraph_mode == "default":
if enforce_eager:
row["Eager Mode"] = CHECK
elif has_cudagraph_sizes:
row["Default FULL_AND_PIECEWISE Graph"] = CHECK
else:
row["Eager Mode"] = CHECK
elif enforce_eager:
row["Eager Mode"] = CHECK
else:
row["Eager Mode"] = EMPTY
has_pd_disagg = False
if kv_transfer_config:
kv_role = kv_transfer_config.get("kv_role", "")
if "kv_producer" in str(kv_role) or "kv_consumer" in str(kv_role):
has_pd_disagg = True
row["PD disaggregation"] = CHECK if has_pd_disagg else EMPTY
is_w8a8 = "w8a8" in model.lower()
is_w4a8 = "w4a8" in model.lower()
row["W8A8"] = CHECK if is_w8a8 else EMPTY
row["W4A8"] = CHECK if is_w4a8 else EMPTY
is_fp16 = dtype in ("float16", "half") or (is_310p and dtype != "bfloat16" and not is_w8a8)
row["FP16"] = CHECK if is_fp16 else EMPTY
row["LoRA"] = CHECK if enable_lora else EMPTY
row["Multi-LoRA"] = CHECK if (max_loras is not None and max_loras > 1) else EMPTY
row["Runtime LoRA updating"] = EMPTY
row["Fully sharded LoRA parameterization"] = CHECK if fully_sharded else EMPTY
has_spec = speculative_config is not None
row["Spec Decode"] = CHECK if has_spec else EMPTY
has_mtp = False
if speculative_config and isinstance(speculative_config, dict):
method = speculative_config.get("method", "")
if method and "mtp" in method.lower():
has_mtp = True
row["MTP"] = CHECK if has_mtp else EMPTY
has_eagle3 = False
if speculative_config and isinstance(speculative_config, dict):
method = speculative_config.get("method", "")
if method and "eagle3" in method.lower():
has_eagle3 = True
row["Eagle-3"] = CHECK if has_eagle3 else EMPTY
has_sfa_dsa = bool(enable_dsa_cp)
if has_sfa_dsa:
row["SFA/DSA"] = CHECK
else:
row["SFA/DSA"] = EMPTY
row["DSA CP"] = EMPTY
row["Pooling runner"] = CHECK if runner == "pooling" else EMPTY
row["Score API"] = EMPTY
row["Classification API"] = EMPTY
row["Distributed executor mp"] = CHECK if dist_exec_mp else EMPTY
has_fa3 = attention_backend and "FLASH_ATTN" in str(attention_backend)
row["Flash Attention 3"] = CHECK if has_fa3 else EMPTY
row["FIA comparison"] = EMPTY
row["Chunked Prefill"] = CHECK if enable_chunked_prefill else EMPTY
row["Prefix Caching"] = CHECK if enable_prefix_caching else EMPTY
row["CPU/KV offloading"] = EMPTY
has_kv_transfer = False
if kv_transfer_config:
connector = kv_transfer_config.get("kv_connector", "")
if connector and connector != "OffloadingConnector" and connector != "ExampleHiddenStatesConnector":
has_kv_transfer = True
row["KV transfer/events"] = CHECK if has_kv_transfer else EMPTY
row["Sleep/Wake memory"] = CHECK if sleep_mode else EMPTY
has_xlite = False
if additional_config and "xlite_graph_config" in additional_config:
has_xlite = True
row["Xlite Graph"] = CHECK if has_xlite else EMPTY
row["CP KV Interleave"] = CHECK if (cp_kv_interleave is not None and cp_kv_interleave > 0) else EMPTY
is_context_parallel = max_model_len is not None and max_model_len > 8192
row["Context Parallel"] = CHECK if is_context_parallel else EMPTY
has_flashcomm1 = env_vars.get("VLLM_ASCEND_ENABLE_FLASHCOMM1") == "1"
if enable_flashcomm1:
has_flashcomm1 = True
row["FlashComm1 env"] = CHECK if has_flashcomm1 else EMPTY
row["Skipped"] = EMPTY
row["Conditional skip"] = EMPTY
row["Logprobs"] = EMPTY
row["Batch inference"] = EMPTY
row["Mixed lengths"] = EMPTY
return row
if "test_name" in data and "model" in data:
envs = dict(data.get("env_common", {}) or data.get("envs", {}))
features: dict[str, Any] = {}
deployment = data.get("deployment", [])
if deployment:
features.update(_parse_server_cmd(deployment[0].get("server_cmd", "")))
if "envs" in deployment[0]:
envs.update(deployment[0]["envs"])
templates = data.get("templates", [])
if templates:
features.update(_parse_server_cmd(templates[0].get("server_cmd_template", [])))
if "envs" in templates[0]:
envs.update(templates[0]["envs"])
features["envs"] = envs
row = _build_row(data["test_name"], data.get("model", ""), features)
results = [row]
elif "test_cases" in data:
results = []
for tc in data["test_cases"]:
features = {}
server_cmd = tc.get("server_cmd", [])
if server_cmd:
features.update(_parse_server_cmd(server_cmd))
features["envs"] = tc.get("envs", {})
row = _build_row(tc.get("name", tc.get("model", "")), tc.get("model", ""), features)
results.append(row)
else:
return []
return results
def main():
rows = []
test_files = sorted(E2E_PR_ROOT.rglob("test_*.py"))
for filepath in test_files:
source_code = filepath.read_text(encoding="utf-8", errors="replace")
file_rows = _process_test_file(filepath, source_code)
rows.extend(file_rows)
rows.sort(key=lambda r: (r["Test file"], r["Test method"]))
card_sections_rows: dict[str, list] = {}
for section_title, card_prefix in CARD_SECTIONS:
card_sections_rows[section_title] = []
for row in rows:
orig_path = row.get("_orig_rel_path", "")
for section_title, card_prefix in CARD_SECTIONS:
if orig_path.startswith(card_prefix + "/"):
card_sections_rows[section_title].append(row)
break
NIGHTLY_ROOT = REPO_ROOT / "tests" / "e2e" / "nightly"
WEEKLY_ROOT = REPO_ROOT / "tests" / "e2e" / "weekly"
def _scan_tests(root):
py_rows = []
yaml_rows = []
for fp in sorted(root.rglob("test_*.py")):
src = fp.read_text(encoding="utf-8", errors="replace")
py_rows.extend(_process_test_file(fp, src, root))
for fp in sorted(root.rglob("*.yaml")):
src = fp.read_text(encoding="utf-8", errors="replace")
yaml_rows.extend(_process_yaml_file(fp, src, root))
combined = py_rows + yaml_rows
combined.sort(key=lambda r: (r["Test file"], r["Test method"]))
return combined
nightly_rows = _scan_tests(NIGHTLY_ROOT)
weekly_rows = _scan_tests(WEEKLY_ROOT)
header = "| " + " | ".join(COLUMNS) + " |"
separator = "| " + " | ".join(["---"] * len(COLUMNS)) + " |"
output = "The coverage of e2e is as follows:\n\n"
for section_title, card_prefix in CARD_SECTIONS:
section_rows = card_sections_rows[section_title]
output += f"## {section_title}\n\n"
output += header + "\n"
output += separator + "\n"
for row in section_rows:
vals = [row.get(col, EMPTY) for col in COLUMNS]
output += "| " + " | ".join(vals) + " |\n"
output += "\n"
for section_title, rows_data in [("Nightly Tests", nightly_rows), ("Weekly Tests", weekly_rows)]:
output += f"## {section_title}\n\n"
output += header + "\n"
output += separator + "\n"
for row in rows_data:
vals = [row.get(col, EMPTY) for col in COLUMNS]
output += "| " + " | ".join(vals) + " |\n"
output += "\n"
print(output)
OUTPUT_FILE.write_text(output.rstrip("\n") + "\n", encoding="utf-8")
print(f"\nWritten to {OUTPUT_FILE}")
if __name__ == "__main__":
main()