1328 lines
49 KiB
Python
1328 lines
49 KiB
Python
#!/usr/bin/env python3
|
|
"""Generate coverage.md by scanning test files (Python + YAML) and
|
|
extracting feature flags via AST and YAML parsing."""
|
|
|
|
import ast
|
|
import contextlib
|
|
import json
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
import regex as re
|
|
|
|
REPO_ROOT = Path(__file__).resolve().parents[2]
|
|
E2E_PR_ROOT = REPO_ROOT / "tests" / "e2e" / "pull_request"
|
|
OUTPUT_FILE = Path(__file__).resolve().parent / "coverage.md"
|
|
|
|
COLUMNS = [
|
|
"Test file",
|
|
"Test method",
|
|
"Model",
|
|
"310P",
|
|
"Dense",
|
|
"MoE",
|
|
"Embedding",
|
|
"Classification",
|
|
"Reranker",
|
|
"Mamba/SSM",
|
|
"Multimodal Reasoning",
|
|
"TP",
|
|
"PP",
|
|
"EP",
|
|
"PCP",
|
|
"DCP",
|
|
"Context Parallel",
|
|
"EPLB",
|
|
"Dynamic EPLB",
|
|
"Multistream MoE",
|
|
"Full Graph",
|
|
"Full Decode Only Graph",
|
|
"Default FULL_AND_PIECEWISE Graph",
|
|
"Piecewise Graph",
|
|
"Eager Mode",
|
|
"PD disaggregation",
|
|
"W8A8",
|
|
"W4A8",
|
|
"FP16",
|
|
"LoRA",
|
|
"Multi-LoRA",
|
|
"Runtime LoRA updating",
|
|
"Fully sharded LoRA parameterization",
|
|
"Spec Decode",
|
|
"MTP",
|
|
"Eagle-3",
|
|
"SFA/DSA",
|
|
"DSA CP",
|
|
"Pooling runner",
|
|
"Score API",
|
|
"Classification API",
|
|
"Distributed executor mp",
|
|
"Flash Attention 3",
|
|
"FIA comparison",
|
|
"Chunked Prefill",
|
|
"Prefix Caching",
|
|
"CPU/KV offloading",
|
|
"KV transfer/events",
|
|
"Sleep/Wake memory",
|
|
"Xlite Graph",
|
|
"CP KV Interleave",
|
|
"Long Sequence",
|
|
"FlashComm1 env",
|
|
"Skipped",
|
|
"Conditional skip",
|
|
"Logprobs",
|
|
"Batch inference",
|
|
"Mixed lengths",
|
|
]
|
|
|
|
CARD_SECTIONS = [
|
|
("1-Card Tests", "one_card"),
|
|
("2-Card Tests", "two_card"),
|
|
("4-Card Tests", "four_card"),
|
|
]
|
|
|
|
CHECK = "\u2705"
|
|
EMPTY = ""
|
|
|
|
|
|
def _source_to_str(node):
|
|
if isinstance(node, ast.Constant):
|
|
return str(node.value)
|
|
if isinstance(node, ast.Str):
|
|
return node.s
|
|
return None
|
|
|
|
|
|
def _flatten_dict_keys(d):
|
|
out = set()
|
|
if isinstance(d, dict):
|
|
for k, v in d.items():
|
|
out.add(k)
|
|
if isinstance(v, dict):
|
|
out.update(_flatten_dict_keys(v))
|
|
return out
|
|
|
|
|
|
def _extract_kwarg_dict(call_node, key):
|
|
for kw in call_node.keywords:
|
|
if kw.arg == key and isinstance(kw.value, ast.Dict):
|
|
d = {}
|
|
for dk, dv in zip(kw.value.keys, kw.value.values):
|
|
ks = _source_to_str(dk)
|
|
vs = _source_to_str(dv)
|
|
if ks is not None:
|
|
d[ks] = vs
|
|
elif isinstance(dk, ast.Constant):
|
|
ks2 = str(dk.value)
|
|
d[ks2] = vs
|
|
return d
|
|
return None
|
|
|
|
|
|
def _extract_kwarg_value(call_node, key):
|
|
for kw in call_node.keywords:
|
|
if kw.arg == key:
|
|
return _source_to_str(kw.value)
|
|
return None
|
|
|
|
|
|
def _extract_kwarg_int(call_node, key):
|
|
for kw in call_node.keywords:
|
|
if kw.arg == key and isinstance(kw.value, ast.Constant):
|
|
try:
|
|
return int(kw.value.value)
|
|
except (TypeError, ValueError):
|
|
return None
|
|
return None
|
|
|
|
|
|
def _extract_kwarg_bool(call_node, key):
|
|
for kw in call_node.keywords:
|
|
if kw.arg == key and isinstance(kw.value, ast.Constant):
|
|
return bool(kw.value.value)
|
|
return None
|
|
|
|
|
|
def _find_calls_in_body(body, func_name):
|
|
calls = []
|
|
for node in _walk_body(body):
|
|
if isinstance(node, ast.Call):
|
|
if (
|
|
isinstance(node.func, ast.Name)
|
|
and node.func.id == func_name
|
|
or isinstance(node.func, ast.Attribute)
|
|
and node.func.attr == func_name
|
|
):
|
|
calls.append(node)
|
|
return calls
|
|
|
|
|
|
def _find_all_calls_in_body(body):
|
|
calls = []
|
|
for node in _walk_body(body):
|
|
if isinstance(node, ast.Call):
|
|
calls.append(node)
|
|
return calls
|
|
|
|
|
|
def _extract_model_from_call(call_node):
|
|
model = _extract_kwarg_value(call_node, "model_name") or _extract_kwarg_value(call_node, "model")
|
|
if model:
|
|
return model
|
|
pos_args = call_node.args
|
|
if pos_args and isinstance(pos_args[0], ast.Constant):
|
|
return str(pos_args[0].value)
|
|
return None
|
|
|
|
|
|
def _extract_config_from_call(call_node):
|
|
config_keys = set()
|
|
for kw in call_node.keywords:
|
|
config_keys.add(kw.arg)
|
|
comp_config = _extract_kwarg_dict(call_node, "compilation_config")
|
|
if comp_config:
|
|
config_keys.update(_flatten_dict_keys(comp_config))
|
|
spec_config = _extract_kwarg_dict(call_node, "speculative_config")
|
|
if spec_config:
|
|
config_keys.update(_flatten_dict_keys(spec_config))
|
|
add_config = _extract_kwarg_dict(call_node, "additional_config")
|
|
if add_config:
|
|
config_keys.update(_flatten_dict_keys(add_config))
|
|
return config_keys
|
|
|
|
|
|
def _detect_cards(rel_path):
|
|
parts = rel_path.split("/")
|
|
for p in parts:
|
|
if p == "one_card":
|
|
return 1
|
|
if p == "two_card":
|
|
return 2
|
|
if p == "four_card":
|
|
return 4
|
|
return 1
|
|
|
|
|
|
def _detect_310p(rel_path):
|
|
return "_310p" in rel_path or "/310p/" in rel_path or rel_path.startswith("310p/")
|
|
|
|
|
|
def _classify_model(model_name):
|
|
flags = {
|
|
"Dense": False,
|
|
"MoE": False,
|
|
"Embedding": False,
|
|
"Classification": False,
|
|
"Reranker": False,
|
|
"Mamba/SSM": False,
|
|
"Multimodal Reasoning": False,
|
|
}
|
|
if not model_name:
|
|
return flags
|
|
ml = model_name.lower()
|
|
is_moe = any(
|
|
x in model_name
|
|
for x in [
|
|
"A3B",
|
|
"MoE",
|
|
"DeepSeek-V2",
|
|
"DeepSeek-V3",
|
|
"DeepSeek-V4",
|
|
"Qwen3-30B-A3B",
|
|
"Qwen3.5-35B-A3B",
|
|
"Qwen3-Next-80B-A3B",
|
|
"Qwen3-Coder-30B-A3B",
|
|
]
|
|
)
|
|
is_embedding = any(x in ml for x in ["embedding", "e5", "bge-m3", "minilm"]) or any(
|
|
x in model_name for x in ["Embedding"]
|
|
)
|
|
is_classification = any(x in ml for x in ["apeach", "classification"]) or "SequenceClassification" in model_name
|
|
is_reranker = any(x in ml for x in ["reranker"])
|
|
is_mamba_ssm = "Qwen3.5" in model_name and "VL" not in model_name
|
|
is_vl = any(x in model_name for x in ["VL", "HunyuanOCR", "Audio", "whisper", "MiniCPM-V"])
|
|
if is_vl:
|
|
flags["Multimodal Reasoning"] = True
|
|
if is_mamba_ssm and not is_vl:
|
|
flags["Mamba/SSM"] = True
|
|
if is_reranker:
|
|
flags["Reranker"] = True
|
|
if is_classification:
|
|
flags["Classification"] = True
|
|
if is_embedding:
|
|
flags["Embedding"] = True
|
|
if is_moe:
|
|
flags["MoE"] = True
|
|
is_other = not (is_moe or is_embedding or is_classification or is_reranker or is_vl)
|
|
if is_other and not is_mamba_ssm:
|
|
flags["Dense"] = True
|
|
return flags
|
|
|
|
|
|
def _extract_env_vars(body):
|
|
env_vars = {}
|
|
for node in _walk_body(body):
|
|
if isinstance(node, ast.Call):
|
|
fn = None
|
|
if isinstance(node.func, ast.Name):
|
|
fn = node.func.id
|
|
elif isinstance(node.func, ast.Attribute):
|
|
fn = node.func.attr
|
|
if fn == "setenv" and node.args:
|
|
key = _source_to_str(node.args[0])
|
|
val = _source_to_str(node.args[1]) if len(node.args) > 1 else None
|
|
if key:
|
|
env_vars[key] = val
|
|
if fn == "patch" and isinstance(node.func, ast.Attribute):
|
|
pass
|
|
for node in _walk_body(body):
|
|
if isinstance(node, ast.Subscript):
|
|
try:
|
|
if (
|
|
(isinstance(node.value, ast.Attribute) and node.value.attr == "environ")
|
|
or isinstance(node.value, ast.Name)
|
|
and node.value.id == "environ"
|
|
):
|
|
sl = node.slice
|
|
if isinstance(sl, ast.Constant):
|
|
env_vars[str(sl.value)] = None
|
|
except Exception:
|
|
pass
|
|
for node in _walk_body(body):
|
|
if isinstance(node, ast.Call) and isinstance(node.func, ast.Attribute):
|
|
if node.func.attr == "dict" and isinstance(node.func.value, ast.Name):
|
|
if node.func.value.id == "patch":
|
|
for kw in node.keywords:
|
|
if kw.arg == "os.environ" and isinstance(kw.value, ast.Dict):
|
|
for dk, dv in zip(kw.value.keys, kw.value.values):
|
|
ks = _source_to_str(dk)
|
|
vs = _source_to_str(dv)
|
|
if ks:
|
|
env_vars[ks] = vs
|
|
return env_vars
|
|
|
|
|
|
def _extract_decorators(func_node):
|
|
has_skip = False
|
|
has_skipif = False
|
|
for dec in func_node.decorator_list:
|
|
if isinstance(dec, ast.Call) and isinstance(dec.func, ast.Name):
|
|
if dec.func.id == "pytest.mark.skip":
|
|
has_skip = True
|
|
if dec.func.id == "pytest.mark.skipif":
|
|
has_skipif = True
|
|
if isinstance(dec, ast.Call) and isinstance(dec.func, ast.Attribute):
|
|
if dec.func.attr == "skip":
|
|
has_skip = True
|
|
if dec.func.attr == "skipif":
|
|
has_skipif = True
|
|
if isinstance(dec, ast.Name) and dec.id == "pytest.mark.skip":
|
|
has_skip = True
|
|
return has_skip, has_skipif
|
|
|
|
|
|
def _walk_body(body):
|
|
if isinstance(body, list):
|
|
for stmt in body:
|
|
yield from ast.walk(stmt)
|
|
else:
|
|
yield from ast.walk(body)
|
|
|
|
|
|
def _find_llm_or_vllmrunner_calls(body):
|
|
calls = []
|
|
for node in _walk_body(body):
|
|
if isinstance(node, ast.Call):
|
|
if (
|
|
isinstance(node.func, ast.Name)
|
|
and node.func.id in ("LLM", "VllmRunner")
|
|
or isinstance(node.func, ast.Name)
|
|
and node.func.id == "compare_logprobs"
|
|
or isinstance(node.func, ast.Name)
|
|
and node.func.id == "check_outputs_equal"
|
|
):
|
|
calls.append(node)
|
|
return calls
|
|
|
|
|
|
def _extract_all_kwarg_keys(body):
|
|
keys = set()
|
|
for node in _walk_body(body):
|
|
if isinstance(node, ast.Call):
|
|
for kw in node.keywords:
|
|
if kw.arg:
|
|
keys.add(kw.arg)
|
|
return keys
|
|
|
|
|
|
def _find_method_calls(body):
|
|
method_names = set()
|
|
for node in _walk_body(body):
|
|
if isinstance(node, ast.Call) and isinstance(node.func, ast.Attribute):
|
|
method_names.add(node.func.attr)
|
|
return method_names
|
|
|
|
|
|
def _find_lora_requests(body):
|
|
count = 0
|
|
for node in _walk_body(body):
|
|
if isinstance(node, ast.Call):
|
|
if isinstance(node.func, ast.Name) and node.func.id == "LoRARequest":
|
|
count += 1
|
|
return count
|
|
|
|
|
|
def _extract_models_from_parametrize(func_node, source_code):
|
|
models = []
|
|
for dec in func_node.decorator_list:
|
|
if isinstance(dec, ast.Call) and isinstance(dec.func, ast.Attribute):
|
|
if dec.func.attr == "parametrize":
|
|
for kw in dec.keywords:
|
|
if kw.arg in ("model", "model_name", "model_setup") and isinstance(kw.value, ast.List):
|
|
for elt in kw.value.elts:
|
|
if isinstance(elt, ast.Constant):
|
|
val = elt.value
|
|
if isinstance(val, str):
|
|
models.append(val)
|
|
elif isinstance(val, tuple):
|
|
models.append(val[1] if len(val) > 1 else str(val))
|
|
elif kw.arg == "method" and isinstance(kw.value, ast.Dict):
|
|
pass
|
|
return models
|
|
|
|
|
|
def _extract_server_args_models(body, source_code):
|
|
models = set()
|
|
for node in _walk_body(body):
|
|
if isinstance(node, ast.Call):
|
|
if isinstance(node.func, ast.Name) and node.func.id in (
|
|
"RemoteOpenAIServer",
|
|
"RemotePDServer",
|
|
"RemoteEPDServer",
|
|
):
|
|
for arg in node.args:
|
|
if isinstance(arg, ast.Constant):
|
|
models.add(str(arg.value))
|
|
for kw in node.keywords:
|
|
if kw.arg == "model" and isinstance(kw.value, ast.Constant):
|
|
models.add(str(kw.value.value))
|
|
if not models:
|
|
docstring_val = None
|
|
if isinstance(body, list) and body and isinstance(body[0], ast.Expr):
|
|
if isinstance(body[0].value, ast.Constant) and isinstance(body[0].value.value, str):
|
|
docstring_val = body[0].value.value
|
|
str_literals = set()
|
|
for node in _walk_body(body):
|
|
if isinstance(node, ast.Constant) and isinstance(node.value, str):
|
|
v = node.value
|
|
if v == docstring_val:
|
|
continue
|
|
if len(v) > 200:
|
|
continue
|
|
if any(
|
|
p in v
|
|
for p in [
|
|
"Qwen",
|
|
"DeepSeek",
|
|
"MiniCPM",
|
|
"ilama",
|
|
"LLM-Research",
|
|
"gdydems",
|
|
"BAAI",
|
|
"intfloat",
|
|
"sentence-transformers",
|
|
"Howeee",
|
|
"vllm-ascend",
|
|
"charent",
|
|
"RedHatAI",
|
|
"z-lab",
|
|
"amd",
|
|
"MNN",
|
|
"wemaster",
|
|
"dengcao",
|
|
"openai-mirror",
|
|
"openbmb",
|
|
"OpenBMB",
|
|
"Meta-Llama",
|
|
"amazon",
|
|
]
|
|
):
|
|
if not v.startswith("--") and "/" in v and len(v) > 5:
|
|
str_literals.add(v)
|
|
models = str_literals
|
|
return models
|
|
|
|
|
|
def _process_test_file(filepath, source_code, root_path=None):
|
|
try:
|
|
tree = ast.parse(source_code)
|
|
except SyntaxError:
|
|
return []
|
|
|
|
if root_path is None:
|
|
root_path = E2E_PR_ROOT
|
|
rel_path = str(filepath.relative_to(root_path))
|
|
is_310p = _detect_310p(rel_path)
|
|
|
|
card_prefix = ""
|
|
for prefix in ("one_card/", "two_card/", "four_card/"):
|
|
if rel_path.startswith(prefix):
|
|
card_prefix = prefix
|
|
break
|
|
display_path = rel_path[len(card_prefix) :] if card_prefix else rel_path
|
|
|
|
results = []
|
|
|
|
for node in ast.walk(tree):
|
|
if not isinstance(node, ast.FunctionDef):
|
|
continue
|
|
if not node.name.startswith("test_"):
|
|
continue
|
|
|
|
test_name = node.name
|
|
has_skip, has_skipif = _extract_decorators(node)
|
|
|
|
all_calls = _find_llm_or_vllmrunner_calls(node.body)
|
|
config_keys = _extract_all_kwarg_keys(node.body)
|
|
kwarg_vals = {}
|
|
for call in all_calls:
|
|
for kw in call.keywords:
|
|
if kw.arg and isinstance(kw.value, ast.Constant):
|
|
kwarg_vals[kw.arg] = kw.value.value
|
|
env_vars = _extract_env_vars(node.body)
|
|
method_calls = _find_method_calls(node.body)
|
|
lora_count = _find_lora_requests(node.body)
|
|
|
|
models_from_params = _extract_models_from_parametrize(node, source_code)
|
|
server_models = _extract_server_args_models(node.body, source_code)
|
|
|
|
all_models = []
|
|
if models_from_params:
|
|
all_models = list(models_from_params)
|
|
elif server_models:
|
|
all_models = sorted(server_models)
|
|
else:
|
|
found_models = set()
|
|
for call in all_calls:
|
|
m = _extract_model_from_call(call)
|
|
if m:
|
|
found_models.add(m)
|
|
for call in all_calls:
|
|
if isinstance(call.func, ast.Name) and call.func.id == "compare_logprobs":
|
|
m = _extract_kwarg_value(call, "model_name")
|
|
if m:
|
|
found_models.add(m)
|
|
for var_name in ("model", "model_name", "MODEL_NAME", "DEFAULT_MODEL", "MODEL_PATH"):
|
|
for assign in _walk_body(node.body):
|
|
if isinstance(assign, ast.Assign):
|
|
for target in assign.targets:
|
|
if isinstance(target, ast.Name) and target.id == var_name:
|
|
val = _source_to_str(assign.value)
|
|
if val and "/" in val:
|
|
found_models.add(val)
|
|
for var_name in ("MODELS", "CROSS_ENCODER_MODELS", "EMBEDDING_MODELS", "WHISPER_MODELS", "MINICPM_MODELS"):
|
|
for assign in ast.walk(tree):
|
|
if isinstance(assign, ast.Assign):
|
|
for target in assign.targets:
|
|
if isinstance(target, ast.Name) and target.id == var_name:
|
|
if isinstance(assign.value, ast.List):
|
|
for elt in assign.value.elts:
|
|
v = _source_to_str(elt)
|
|
if v and "/" in v:
|
|
found_models.add(v)
|
|
if found_models:
|
|
all_models = sorted(found_models)
|
|
|
|
if not all_models:
|
|
model_str = "-"
|
|
else:
|
|
model_str = "<br>".join(all_models)
|
|
|
|
model_flags = {}
|
|
for m in all_models:
|
|
mf = _classify_model(m)
|
|
for k, v in mf.items():
|
|
if v:
|
|
model_flags[k] = True
|
|
|
|
if not model_flags and not all_models:
|
|
model_flags = {}
|
|
|
|
tp = _extract_kwarg_int_from_all(all_calls, "tensor_parallel_size")
|
|
pp = _extract_kwarg_int_from_all(all_calls, "pipeline_parallel_size")
|
|
ep = _extract_kwarg_bool_from_all(all_calls, "enable_expert_parallel")
|
|
pcp = _extract_kwarg_int_from_all(all_calls, "prefill_context_parallel_size")
|
|
dcp = _extract_kwarg_int_from_all(all_calls, "decode_context_parallel_size")
|
|
enforce_eager = _extract_kwarg_bool_from_all(all_calls, "enforce_eager")
|
|
dist_exec_mp = _extract_kwarg_bool_from_all(all_calls, "distributed_executor_backend")
|
|
|
|
cudagraph_mode = _extract_cudagraph_mode(all_calls)
|
|
has_cudagraph_sizes = _has_cudagraph_capture_sizes(all_calls)
|
|
|
|
enable_lora = _extract_kwarg_bool_from_all(all_calls, "enable_lora")
|
|
max_loras = _extract_kwarg_int_from_all(all_calls, "max_loras")
|
|
fully_sharded = _extract_kwarg_bool_from_all(all_calls, "fully_sharded_loras")
|
|
|
|
spec_config = None
|
|
for call in all_calls:
|
|
sc = _extract_kwarg_dict(call, "speculative_config")
|
|
if sc:
|
|
spec_config = sc
|
|
|
|
kv_transfer_config = None
|
|
for call in all_calls:
|
|
kc = _extract_kwarg_dict(call, "kv_transfer_config")
|
|
if kc:
|
|
kv_transfer_config = kc
|
|
|
|
kv_events_config = None
|
|
for call in all_calls:
|
|
kec = _extract_kwarg_dict(call, "kv_events_config")
|
|
if kec:
|
|
kv_events_config = kec
|
|
|
|
runner = _extract_kwarg_value_from_all(all_calls, "runner")
|
|
enable_prefix_caching = _extract_kwarg_bool_from_all(all_calls, "enable_prefix_caching")
|
|
enable_chunked_prefill = _extract_kwarg_bool_from_all(all_calls, "enable_chunked_prefill")
|
|
cp_kv_interleave = _extract_kwarg_int_from_all(all_calls, "cp_kv_cache_interleave_size")
|
|
max_model_len = _extract_kwarg_int_from_all(all_calls, "max_model_len")
|
|
limit_mm = _extract_kwarg_value_from_all(all_calls, "limit_mm_per_prompt")
|
|
attention_backend = _extract_kwarg_value_from_all(all_calls, "attention_backend")
|
|
quantization = _extract_kwarg_value_from_all(all_calls, "quantization")
|
|
dtype = _extract_kwarg_value_from_all(all_calls, "dtype")
|
|
sleep_mode = _extract_kwarg_bool_from_all(all_calls, "enable_sleep_mode")
|
|
|
|
add_config = None
|
|
for call in all_calls:
|
|
ac = _extract_kwarg_dict(call, "additional_config")
|
|
if ac:
|
|
add_config = ac
|
|
|
|
for call in all_calls:
|
|
cc = _extract_kwarg_dict(call, "compilation_config")
|
|
if cc:
|
|
break
|
|
|
|
xlite_graph = None
|
|
for call in all_calls:
|
|
xc = _extract_kwarg_dict(call, "additional_config")
|
|
if xc and "xlite_graph_config" in xc:
|
|
xlite_graph = xc["xlite_graph_config"]
|
|
|
|
enable_dsa_cp = None
|
|
multistream_moe = None
|
|
enable_flashcomm1 = None
|
|
if add_config:
|
|
if "enable_dsa_cp" in add_config:
|
|
enable_dsa_cp = add_config["enable_dsa_cp"]
|
|
if "enable_multistream_moe" in add_config:
|
|
multistream_moe = add_config["enable_multistream_moe"]
|
|
if "multistream_overlap_shared_expert" in add_config:
|
|
multistream_moe = add_config["multistream_overlap_shared_expert"]
|
|
if "enable_flashcomm1" in add_config:
|
|
enable_flashcomm1 = add_config["enable_flashcomm1"]
|
|
|
|
eplb_config = None
|
|
for call in all_calls:
|
|
ec = _extract_kwarg_dict(call, "additional_config")
|
|
if ec and "eplb_config" in ec:
|
|
eplb_config = ec["eplb_config"]
|
|
|
|
has_mamba_ssm_dtype = "mamba_ssm_cache_dtype" in config_keys
|
|
|
|
row = {}
|
|
|
|
row["Test file"] = display_path
|
|
row["_orig_rel_path"] = rel_path
|
|
row["Test method"] = test_name
|
|
row["Model"] = model_str
|
|
row["310P"] = CHECK if is_310p else EMPTY
|
|
|
|
row["Dense"] = CHECK if model_flags.get("Dense") else EMPTY
|
|
row["MoE"] = CHECK if model_flags.get("MoE") else EMPTY
|
|
row["Embedding"] = CHECK if model_flags.get("Embedding") else EMPTY
|
|
row["Classification"] = CHECK if model_flags.get("Classification") else EMPTY
|
|
row["Reranker"] = CHECK if model_flags.get("Reranker") else EMPTY
|
|
row["Mamba/SSM"] = CHECK if (model_flags.get("Mamba/SSM") or has_mamba_ssm_dtype) else EMPTY
|
|
row["Multimodal Reasoning"] = CHECK if (model_flags.get("Multimodal Reasoning") or limit_mm) else EMPTY
|
|
|
|
row["TP"] = CHECK if (tp is not None and tp > 1) else EMPTY
|
|
row["PP"] = CHECK if (pp is not None and pp > 1) else EMPTY
|
|
row["EP"] = CHECK if ep else EMPTY
|
|
row["PCP"] = CHECK if (pcp is not None and pcp > 1) else EMPTY
|
|
row["DCP"] = CHECK if (dcp is not None and dcp > 1) else EMPTY
|
|
row["Context Parallel"] = CHECK if ((pcp is not None and pcp > 1) or (dcp is not None and dcp > 1)) else EMPTY
|
|
|
|
has_eplb = (
|
|
eplb_config is not None or "expert_parallel_load_balancing" in config_keys or "eplb_config" in config_keys
|
|
)
|
|
row["EPLB"] = CHECK if has_eplb else EMPTY
|
|
|
|
has_dynamic_eplb = False
|
|
if eplb_config and isinstance(eplb_config, dict):
|
|
has_dynamic_eplb = eplb_config.get("dynamic_eplb") or "eplb_policy_type" in eplb_config
|
|
if "dynamic_eplb" in config_keys or "DYNAMIC_EPLB" in env_vars:
|
|
has_dynamic_eplb = True
|
|
row["Dynamic EPLB"] = CHECK if has_dynamic_eplb else EMPTY
|
|
|
|
row["Multistream MoE"] = CHECK if multistream_moe else EMPTY
|
|
|
|
if cudagraph_mode == "FULL":
|
|
row["Full Graph"] = CHECK
|
|
elif cudagraph_mode == "FULL_DECODE_ONLY":
|
|
row["Full Decode Only Graph"] = CHECK
|
|
elif cudagraph_mode == "PIECEWISE":
|
|
row["Piecewise Graph"] = CHECK
|
|
elif cudagraph_mode == "default":
|
|
if enforce_eager:
|
|
row["Eager Mode"] = CHECK
|
|
elif has_cudagraph_sizes:
|
|
row["Default FULL_AND_PIECEWISE Graph"] = CHECK
|
|
else:
|
|
row["Eager Mode"] = CHECK
|
|
elif enforce_eager:
|
|
row["Eager Mode"] = CHECK
|
|
else:
|
|
row["Eager Mode"] = EMPTY
|
|
|
|
has_pd_disagg = False
|
|
if kv_transfer_config:
|
|
kv_role = kv_transfer_config.get("kv_role")
|
|
if kv_role and ("kv_producer" in str(kv_role) or "kv_consumer" in str(kv_role)):
|
|
has_pd_disagg = True
|
|
if "pd_disaggregation" in config_keys:
|
|
has_pd_disagg = True
|
|
for node2 in _walk_body(node.body):
|
|
if isinstance(node2, ast.Name) and node2.id in ("RemotePDServer", "DisaggPDProxy"):
|
|
has_pd_disagg = True
|
|
row["PD disaggregation"] = CHECK if has_pd_disagg else EMPTY
|
|
|
|
is_w8a8 = any("w8a8" in m.lower() for m in all_models) or (
|
|
quantization == "ascend" and any("w8a8" in m.lower() for m in all_models)
|
|
)
|
|
is_w4a8 = any("w4a8" in m.lower() for m in all_models) or (
|
|
quantization == "ascend" and any("w4a8" in m.lower() for m in all_models)
|
|
)
|
|
row["W8A8"] = CHECK if is_w8a8 else EMPTY
|
|
row["W4A8"] = CHECK if is_w4a8 else EMPTY
|
|
|
|
is_fp16 = dtype in ("float16", "half") or (is_310p and dtype != "bfloat16" and not is_w8a8)
|
|
row["FP16"] = CHECK if is_fp16 else EMPTY
|
|
|
|
row["LoRA"] = CHECK if enable_lora else EMPTY
|
|
row["Multi-LoRA"] = (
|
|
CHECK if (lora_count > 2 or (max_loras is not None and max_loras > 1 and lora_count >= 2)) else EMPTY
|
|
)
|
|
|
|
has_runtime_lora = False
|
|
for mn in method_calls:
|
|
if mn in ("add_lora", "remove_lora", "reload_lora"):
|
|
has_runtime_lora = True
|
|
if "VLLM_ALLOW_RUNTIME_LORA_UPDATING" in env_vars:
|
|
has_runtime_lora = True
|
|
row["Runtime LoRA updating"] = CHECK if has_runtime_lora else EMPTY
|
|
row["Fully sharded LoRA parameterization"] = CHECK if fully_sharded else EMPTY
|
|
|
|
has_spec = spec_config is not None or "speculative_config" in config_keys or "speculative_method" in config_keys
|
|
row["Spec Decode"] = CHECK if has_spec else EMPTY
|
|
|
|
has_mtp = False
|
|
if spec_config and isinstance(spec_config, dict):
|
|
method = spec_config.get("method", "")
|
|
if method and "mtp" in method.lower():
|
|
has_mtp = True
|
|
if method in ("deepseek_mtp", "qwen3_5_mtp", "qwen3_next_mtp"):
|
|
has_mtp = True
|
|
if "mtp" in test_name.lower():
|
|
has_mtp = True
|
|
row["MTP"] = CHECK if has_mtp else EMPTY
|
|
|
|
has_eagle3 = False
|
|
if spec_config and isinstance(spec_config, dict):
|
|
method = spec_config.get("method", "")
|
|
if method and ("eagle3" in method.lower() or method == "eagle"):
|
|
spec_model = spec_config.get("model") or ""
|
|
if "eagle3" in spec_model.lower() or method == "eagle3":
|
|
has_eagle3 = True
|
|
for m in all_models:
|
|
if "eagle3" in m.lower():
|
|
has_eagle3 = True
|
|
row["Eagle-3"] = CHECK if has_eagle3 else EMPTY
|
|
|
|
has_sfa_dsa = False
|
|
if (
|
|
enable_dsa_cp
|
|
or "enable_dsa_cp" in config_keys
|
|
or "sfa" in str(add_config).lower()
|
|
or "dsa" in str(add_config).lower()
|
|
):
|
|
has_sfa_dsa = True
|
|
if has_sfa_dsa:
|
|
row["SFA/DSA"] = CHECK
|
|
else:
|
|
row["SFA/DSA"] = EMPTY
|
|
|
|
has_dsa_cp = enable_dsa_cp and ((pcp is not None and pcp > 1) or (dcp is not None and dcp > 1))
|
|
row["DSA CP"] = CHECK if has_dsa_cp else EMPTY
|
|
|
|
row["Pooling runner"] = (
|
|
CHECK if (runner == "pooling" or "task" in config_keys and kwarg_vals.get("task") == "pooling") else EMPTY
|
|
)
|
|
row["Score API"] = CHECK if "score" in method_calls else EMPTY
|
|
row["Classification API"] = CHECK if "classify" in method_calls else EMPTY
|
|
|
|
row["Distributed executor mp"] = CHECK if dist_exec_mp else EMPTY
|
|
|
|
has_fa3 = False
|
|
if attention_backend and "FLASH_ATTN" in str(attention_backend):
|
|
has_fa3 = True
|
|
if "FA3" in str(attention_backend):
|
|
has_fa3 = True
|
|
if "FLASH_ATTN" in str(attention_backend):
|
|
has_fa3 = True
|
|
row["Flash Attention 3"] = CHECK if has_fa3 else EMPTY
|
|
|
|
has_fia = False
|
|
if "FIA" in test_name or "fia" in source_code.lower()[:5000]:
|
|
has_fia = True
|
|
for m in method_calls:
|
|
if "_assert_outputs_match" in m or "_generate_with_backend" in m:
|
|
has_fia = True
|
|
has_fa3_comparison = has_fa3 and has_fia
|
|
row["FIA comparison"] = CHECK if has_fa3_comparison else EMPTY
|
|
|
|
row["Chunked Prefill"] = CHECK if (enable_chunked_prefill or "enable_chunked_prefill" in config_keys) else EMPTY
|
|
row["Prefix Caching"] = CHECK if (enable_prefix_caching or "enable_prefix_caching" in config_keys) else EMPTY
|
|
|
|
has_cpu_offloading = False
|
|
if "OffloadingConnector" in source_code or "cpu_offloading" in config_keys or "kv_connector" in config_keys:
|
|
for call in all_calls:
|
|
kc = _extract_kwarg_dict(call, "kv_transfer_config")
|
|
if kc and kc.get("kv_connector") == "OffloadingConnector":
|
|
has_cpu_offloading = True
|
|
row["CPU/KV offloading"] = CHECK if has_cpu_offloading else EMPTY
|
|
|
|
has_kv_transfer = False
|
|
if kv_transfer_config:
|
|
connector = kv_transfer_config.get("kv_connector")
|
|
if connector and connector != "OffloadingConnector" and connector != "ExampleHiddenStatesConnector":
|
|
has_kv_transfer = True
|
|
if kv_events_config:
|
|
has_kv_transfer = True
|
|
row["KV transfer/events"] = CHECK if has_kv_transfer else EMPTY
|
|
|
|
row["Sleep/Wake memory"] = CHECK if sleep_mode else EMPTY
|
|
row["Xlite Graph"] = CHECK if xlite_graph else EMPTY
|
|
row["CP KV Interleave"] = CHECK if (cp_kv_interleave is not None and cp_kv_interleave > 0) else EMPTY
|
|
|
|
is_context_parallel = (
|
|
(max_model_len is not None and max_model_len > 8192)
|
|
or "context_parallel" in rel_path
|
|
or "context_parallel" in test_name
|
|
)
|
|
row["Context Parallel"] = CHECK if is_context_parallel else EMPTY
|
|
|
|
has_flashcomm1 = (
|
|
"VLLM_ASCEND_ENABLE_FLASHCOMM1" in env_vars and env_vars.get("VLLM_ASCEND_ENABLE_FLASHCOMM1") == "1"
|
|
)
|
|
if enable_flashcomm1:
|
|
has_flashcomm1 = True
|
|
row["FlashComm1 env"] = CHECK if has_flashcomm1 else EMPTY
|
|
|
|
row["Skipped"] = CHECK if has_skip else EMPTY
|
|
row["Conditional skip"] = CHECK if has_skipif else EMPTY
|
|
|
|
has_logprobs = "logprobs" in config_keys or "prompt_logprobs" in config_keys or "num_logprobs" in config_keys
|
|
if "logprobs" in method_calls or "generate_greedy_logprobs" in method_calls:
|
|
has_logprobs = True
|
|
if "compare_logprobs" in method_calls:
|
|
has_logprobs = True
|
|
row["Logprobs"] = CHECK if has_logprobs else EMPTY
|
|
|
|
has_batch = False
|
|
for node2 in _walk_body(node.body):
|
|
if isinstance(node2, ast.List):
|
|
if len(node2.elts) > 1:
|
|
for elt in node2.elts:
|
|
if isinstance(elt, ast.Constant) and isinstance(elt.value, str):
|
|
if len(elt.value) > 5:
|
|
has_batch = True
|
|
if "max_num_seqs" in config_keys:
|
|
mn_seqs = kwarg_vals.get("max_num_seqs")
|
|
if mn_seqs and mn_seqs > 1:
|
|
has_batch = True
|
|
if "batch" in test_name.lower():
|
|
has_batch = True
|
|
row["Batch inference"] = CHECK if has_batch else EMPTY
|
|
|
|
has_mixed_lengths = (
|
|
"mixed" in test_name.lower() or "mixed_length" in config_keys or "mixed_lengths" in test_name.lower()
|
|
)
|
|
for node2 in _walk_body(node.body):
|
|
if isinstance(node2, ast.Constant) and isinstance(node2.value, str):
|
|
if "mixed" in node2.value.lower() and "length" in node2.value.lower():
|
|
has_mixed_lengths = True
|
|
row["Mixed lengths"] = CHECK if has_mixed_lengths else EMPTY
|
|
|
|
results.append(row)
|
|
|
|
return results
|
|
|
|
|
|
def _extract_kwarg_int_from_all(calls, key):
|
|
for call in calls:
|
|
v = _extract_kwarg_int(call, key)
|
|
if v is not None:
|
|
return v
|
|
return None
|
|
|
|
|
|
def _extract_kwarg_bool_from_all(calls, key):
|
|
for call in calls:
|
|
v = _extract_kwarg_bool(call, key)
|
|
if v is not None:
|
|
return v
|
|
return None
|
|
|
|
|
|
def _extract_kwarg_value_from_all(calls, key):
|
|
for call in calls:
|
|
v = _extract_kwarg_value(call, key)
|
|
if v is not None:
|
|
return v
|
|
return None
|
|
|
|
|
|
def _extract_cudagraph_mode(all_calls):
|
|
for call in all_calls:
|
|
cc = _extract_kwarg_dict(call, "compilation_config")
|
|
if cc:
|
|
mode = cc.get("cudagraph_mode")
|
|
if mode:
|
|
cudagraph_mode_map = {
|
|
"FULL_DECODE_ONLY": "FULL_DECODE_ONLY",
|
|
"PIECEWISE": "PIECEWISE",
|
|
"FULL": "FULL",
|
|
}
|
|
return cudagraph_mode_map[mode]
|
|
cg_mode = _extract_kwarg_value(call, "cudagraph_mode")
|
|
if cg_mode:
|
|
return cg_mode
|
|
return "default"
|
|
|
|
|
|
def _has_cudagraph_capture_sizes(all_calls):
|
|
for call in all_calls:
|
|
cc = _extract_kwarg_dict(call, "compilation_config")
|
|
if cc and "cudagraph_capture_sizes" in cc:
|
|
return True
|
|
if _extract_kwarg_value(call, "cudagraph_capture_sizes") is not None:
|
|
return True
|
|
return False
|
|
|
|
|
|
def _parse_server_cmd(server_cmd):
|
|
features: dict[str, Any] = {}
|
|
if isinstance(server_cmd, str):
|
|
args = server_cmd.split()
|
|
elif isinstance(server_cmd, list):
|
|
args = [str(a) for a in server_cmd]
|
|
else:
|
|
return features
|
|
|
|
def _try_json(raw):
|
|
if isinstance(raw, str):
|
|
raw = raw.strip("'").strip('"')
|
|
try:
|
|
return json.loads(raw)
|
|
except (json.JSONDecodeError, ValueError):
|
|
pass
|
|
return None
|
|
|
|
JSON_FLAGS = {
|
|
"--compilation-config": "compilation_config",
|
|
"--speculative-config": "speculative_config",
|
|
"--kv-transfer-config": "kv_transfer_config",
|
|
"--additional-config": "additional_config",
|
|
}
|
|
|
|
i = 0
|
|
while i < len(args):
|
|
a = args[i]
|
|
if a == "--tensor-parallel-size" and i + 1 < len(args):
|
|
with contextlib.suppress(ValueError):
|
|
features["tp"] = int(args[i + 1])
|
|
i += 2
|
|
elif a == "--pipeline-parallel-size" and i + 1 < len(args):
|
|
with contextlib.suppress(ValueError):
|
|
features["pp"] = int(args[i + 1])
|
|
i += 2
|
|
elif a == "--enforce-eager":
|
|
features["enforce_eager"] = True
|
|
i += 1
|
|
elif a == "--enable-expert-parallel":
|
|
features["ep"] = True
|
|
i += 1
|
|
elif a in ("--quantization", "--dtype", "--attention-backend", "--runner") and i + 1 < len(args):
|
|
features[a.removeprefix("--").replace("-", "_")] = args[i + 1]
|
|
i += 2
|
|
elif a in JSON_FLAGS and i + 1 < len(args):
|
|
val = _try_json(args[i + 1])
|
|
if val is not None:
|
|
features[JSON_FLAGS[a]] = val
|
|
i += 2
|
|
elif a == "--enable-prefix-caching":
|
|
features["enable_prefix_caching"] = True
|
|
i += 1
|
|
elif a == "--no-enable-prefix-caching":
|
|
features["enable_prefix_caching"] = False
|
|
i += 1
|
|
elif a == "--enable-chunked-prefill":
|
|
features["enable_chunked_prefill"] = True
|
|
i += 1
|
|
elif a == "--max-model-len" and i + 1 < len(args):
|
|
with contextlib.suppress(ValueError):
|
|
features["max_model_len"] = int(args[i + 1])
|
|
i += 2
|
|
elif a == "--cp-kv-cache-interleave-size" and i + 1 < len(args):
|
|
with contextlib.suppress(ValueError):
|
|
features["cp_kv_interleave"] = int(args[i + 1])
|
|
i += 2
|
|
elif a == "--enable-sleep-mode":
|
|
features["enable_sleep_mode"] = True
|
|
i += 1
|
|
elif a == "--enable-lora":
|
|
features["enable_lora"] = True
|
|
i += 1
|
|
elif a == "--max-loras" and i + 1 < len(args):
|
|
with contextlib.suppress(ValueError):
|
|
features["max_loras"] = int(args[i + 1])
|
|
pass
|
|
i += 2
|
|
elif a == "--fully-sharded-loras":
|
|
features["fully_sharded_loras"] = True
|
|
i += 1
|
|
elif a == "--distributed-executor-backend" and i + 1 < len(args):
|
|
features["dist_exec_mp"] = True
|
|
i += 2
|
|
else:
|
|
i += 1
|
|
|
|
if isinstance(server_cmd, str):
|
|
for flag, key in JSON_FLAGS.items():
|
|
if key not in features:
|
|
m = re.search(rf"{flag}\s+'([^']+)'", server_cmd)
|
|
if m:
|
|
val = _try_json(m.group(1))
|
|
if val is not None:
|
|
features[key] = val
|
|
|
|
return features
|
|
|
|
|
|
def _process_yaml_file(filepath, source_code, root_path):
|
|
try:
|
|
import yaml
|
|
|
|
data = yaml.safe_load(source_code)
|
|
except Exception:
|
|
return []
|
|
if not isinstance(data, dict):
|
|
return []
|
|
|
|
rel_path = str(filepath.relative_to(root_path))
|
|
is_310p = _detect_310p(rel_path)
|
|
|
|
display_path = rel_path
|
|
|
|
def _build_row(test_name, model, features):
|
|
model_flags = _classify_model(model) if model else {}
|
|
tp = features.get("tp")
|
|
pp = features.get("pp")
|
|
ep = features.get("ep")
|
|
enforce_eager = features.get("enforce_eager")
|
|
dtype = features.get("dtype")
|
|
compilation_config = features.get("compilation_config")
|
|
speculative_config = features.get("speculative_config")
|
|
kv_transfer_config = features.get("kv_transfer_config")
|
|
additional_config = features.get("additional_config")
|
|
enable_prefix_caching = features.get("enable_prefix_caching")
|
|
enable_chunked_prefill = features.get("enable_chunked_prefill")
|
|
cp_kv_interleave = features.get("cp_kv_interleave")
|
|
max_model_len = features.get("max_model_len")
|
|
enable_lora = features.get("enable_lora")
|
|
max_loras = features.get("max_loras")
|
|
fully_sharded = features.get("fully_sharded_loras")
|
|
attention_backend = features.get("attention_backend")
|
|
runner = features.get("runner")
|
|
sleep_mode = features.get("enable_sleep_mode")
|
|
dist_exec_mp = features.get("dist_exec_mp")
|
|
env_vars = features.get("envs", {})
|
|
|
|
cudagraph_mode = "default"
|
|
has_cudagraph_sizes = False
|
|
if compilation_config:
|
|
mode = compilation_config.get("cudagraph_mode", "")
|
|
if mode:
|
|
cudagraph_mode = mode
|
|
if "cudagraph_capture_sizes" in compilation_config:
|
|
has_cudagraph_sizes = True
|
|
|
|
eplb_config = None
|
|
multistream_moe = None
|
|
enable_dsa_cp = None
|
|
enable_flashcomm1 = None
|
|
if additional_config:
|
|
if "eplb_config" in additional_config:
|
|
eplb_config = additional_config["eplb_config"]
|
|
if "enable_multistream_moe" in additional_config:
|
|
multistream_moe = additional_config["enable_multistream_moe"]
|
|
if "multistream_overlap_shared_expert" in additional_config:
|
|
multistream_moe = additional_config["multistream_overlap_shared_expert"]
|
|
if "enable_dsa_cp" in additional_config:
|
|
enable_dsa_cp = additional_config["enable_dsa_cp"]
|
|
if "enable_flashcomm1" in additional_config:
|
|
enable_flashcomm1 = additional_config["enable_flashcomm1"]
|
|
if "ascend_compilation_config" in additional_config:
|
|
ascend_cc = additional_config["ascend_compilation_config"]
|
|
if ascend_cc.get("enable_npugraph_ex"):
|
|
has_cudagraph_sizes = True
|
|
|
|
row = {}
|
|
row["Test file"] = display_path
|
|
row["_orig_rel_path"] = rel_path
|
|
row["Test method"] = test_name
|
|
row["Model"] = model if model else "-"
|
|
row["310P"] = CHECK if is_310p else EMPTY
|
|
|
|
row["Dense"] = CHECK if model_flags.get("Dense") else EMPTY
|
|
row["MoE"] = CHECK if model_flags.get("MoE") else EMPTY
|
|
row["Embedding"] = CHECK if model_flags.get("Embedding") else EMPTY
|
|
row["Classification"] = CHECK if model_flags.get("Classification") else EMPTY
|
|
row["Reranker"] = CHECK if model_flags.get("Reranker") else EMPTY
|
|
row["Mamba/SSM"] = CHECK if model_flags.get("Mamba/SSM") else EMPTY
|
|
row["Multimodal Reasoning"] = CHECK if model_flags.get("Multimodal Reasoning") else EMPTY
|
|
|
|
row["TP"] = CHECK if (tp is not None and tp > 1) else EMPTY
|
|
row["PP"] = CHECK if (pp is not None and pp > 1) else EMPTY
|
|
row["EP"] = CHECK if ep else EMPTY
|
|
row["PCP"] = EMPTY
|
|
row["DCP"] = EMPTY
|
|
row["Context Parallel"] = EMPTY
|
|
row["EPLB"] = CHECK if eplb_config else EMPTY
|
|
row["Dynamic EPLB"] = EMPTY
|
|
row["Multistream MoE"] = CHECK if multistream_moe else EMPTY
|
|
|
|
if cudagraph_mode == "FULL":
|
|
row["Full Graph"] = CHECK
|
|
elif cudagraph_mode == "FULL_DECODE_ONLY":
|
|
row["Full Decode Only Graph"] = CHECK
|
|
elif cudagraph_mode == "PIECEWISE":
|
|
row["Piecewise Graph"] = CHECK
|
|
elif cudagraph_mode == "default":
|
|
if enforce_eager:
|
|
row["Eager Mode"] = CHECK
|
|
elif has_cudagraph_sizes:
|
|
row["Default FULL_AND_PIECEWISE Graph"] = CHECK
|
|
else:
|
|
row["Eager Mode"] = CHECK
|
|
elif enforce_eager:
|
|
row["Eager Mode"] = CHECK
|
|
else:
|
|
row["Eager Mode"] = EMPTY
|
|
|
|
has_pd_disagg = False
|
|
if kv_transfer_config:
|
|
kv_role = kv_transfer_config.get("kv_role", "")
|
|
if "kv_producer" in str(kv_role) or "kv_consumer" in str(kv_role):
|
|
has_pd_disagg = True
|
|
row["PD disaggregation"] = CHECK if has_pd_disagg else EMPTY
|
|
|
|
is_w8a8 = "w8a8" in model.lower()
|
|
is_w4a8 = "w4a8" in model.lower()
|
|
row["W8A8"] = CHECK if is_w8a8 else EMPTY
|
|
row["W4A8"] = CHECK if is_w4a8 else EMPTY
|
|
|
|
is_fp16 = dtype in ("float16", "half") or (is_310p and dtype != "bfloat16" and not is_w8a8)
|
|
row["FP16"] = CHECK if is_fp16 else EMPTY
|
|
|
|
row["LoRA"] = CHECK if enable_lora else EMPTY
|
|
row["Multi-LoRA"] = CHECK if (max_loras is not None and max_loras > 1) else EMPTY
|
|
row["Runtime LoRA updating"] = EMPTY
|
|
row["Fully sharded LoRA parameterization"] = CHECK if fully_sharded else EMPTY
|
|
|
|
has_spec = speculative_config is not None
|
|
row["Spec Decode"] = CHECK if has_spec else EMPTY
|
|
|
|
has_mtp = False
|
|
if speculative_config and isinstance(speculative_config, dict):
|
|
method = speculative_config.get("method", "")
|
|
if method and "mtp" in method.lower():
|
|
has_mtp = True
|
|
row["MTP"] = CHECK if has_mtp else EMPTY
|
|
|
|
has_eagle3 = False
|
|
if speculative_config and isinstance(speculative_config, dict):
|
|
method = speculative_config.get("method", "")
|
|
if method and "eagle3" in method.lower():
|
|
has_eagle3 = True
|
|
row["Eagle-3"] = CHECK if has_eagle3 else EMPTY
|
|
|
|
has_sfa_dsa = bool(enable_dsa_cp)
|
|
if has_sfa_dsa:
|
|
row["SFA/DSA"] = CHECK
|
|
else:
|
|
row["SFA/DSA"] = EMPTY
|
|
row["DSA CP"] = EMPTY
|
|
|
|
row["Pooling runner"] = CHECK if runner == "pooling" else EMPTY
|
|
row["Score API"] = EMPTY
|
|
row["Classification API"] = EMPTY
|
|
row["Distributed executor mp"] = CHECK if dist_exec_mp else EMPTY
|
|
|
|
has_fa3 = attention_backend and "FLASH_ATTN" in str(attention_backend)
|
|
row["Flash Attention 3"] = CHECK if has_fa3 else EMPTY
|
|
row["FIA comparison"] = EMPTY
|
|
|
|
row["Chunked Prefill"] = CHECK if enable_chunked_prefill else EMPTY
|
|
row["Prefix Caching"] = CHECK if enable_prefix_caching else EMPTY
|
|
|
|
row["CPU/KV offloading"] = EMPTY
|
|
|
|
has_kv_transfer = False
|
|
if kv_transfer_config:
|
|
connector = kv_transfer_config.get("kv_connector", "")
|
|
if connector and connector != "OffloadingConnector" and connector != "ExampleHiddenStatesConnector":
|
|
has_kv_transfer = True
|
|
row["KV transfer/events"] = CHECK if has_kv_transfer else EMPTY
|
|
|
|
row["Sleep/Wake memory"] = CHECK if sleep_mode else EMPTY
|
|
|
|
has_xlite = False
|
|
if additional_config and "xlite_graph_config" in additional_config:
|
|
has_xlite = True
|
|
row["Xlite Graph"] = CHECK if has_xlite else EMPTY
|
|
|
|
row["CP KV Interleave"] = CHECK if (cp_kv_interleave is not None and cp_kv_interleave > 0) else EMPTY
|
|
|
|
is_context_parallel = max_model_len is not None and max_model_len > 8192
|
|
row["Context Parallel"] = CHECK if is_context_parallel else EMPTY
|
|
|
|
has_flashcomm1 = env_vars.get("VLLM_ASCEND_ENABLE_FLASHCOMM1") == "1"
|
|
if enable_flashcomm1:
|
|
has_flashcomm1 = True
|
|
row["FlashComm1 env"] = CHECK if has_flashcomm1 else EMPTY
|
|
|
|
row["Skipped"] = EMPTY
|
|
row["Conditional skip"] = EMPTY
|
|
row["Logprobs"] = EMPTY
|
|
row["Batch inference"] = EMPTY
|
|
row["Mixed lengths"] = EMPTY
|
|
|
|
return row
|
|
|
|
if "test_name" in data and "model" in data:
|
|
envs = dict(data.get("env_common", {}) or data.get("envs", {}))
|
|
features: dict[str, Any] = {}
|
|
deployment = data.get("deployment", [])
|
|
if deployment:
|
|
features.update(_parse_server_cmd(deployment[0].get("server_cmd", "")))
|
|
if "envs" in deployment[0]:
|
|
envs.update(deployment[0]["envs"])
|
|
templates = data.get("templates", [])
|
|
if templates:
|
|
features.update(_parse_server_cmd(templates[0].get("server_cmd_template", [])))
|
|
if "envs" in templates[0]:
|
|
envs.update(templates[0]["envs"])
|
|
features["envs"] = envs
|
|
row = _build_row(data["test_name"], data.get("model", ""), features)
|
|
results = [row]
|
|
|
|
elif "test_cases" in data:
|
|
results = []
|
|
for tc in data["test_cases"]:
|
|
features = {}
|
|
server_cmd = tc.get("server_cmd", [])
|
|
if server_cmd:
|
|
features.update(_parse_server_cmd(server_cmd))
|
|
features["envs"] = tc.get("envs", {})
|
|
row = _build_row(tc.get("name", tc.get("model", "")), tc.get("model", ""), features)
|
|
results.append(row)
|
|
else:
|
|
return []
|
|
|
|
return results
|
|
|
|
|
|
def main():
|
|
rows = []
|
|
test_files = sorted(E2E_PR_ROOT.rglob("test_*.py"))
|
|
|
|
for filepath in test_files:
|
|
source_code = filepath.read_text(encoding="utf-8", errors="replace")
|
|
file_rows = _process_test_file(filepath, source_code)
|
|
rows.extend(file_rows)
|
|
|
|
rows.sort(key=lambda r: (r["Test file"], r["Test method"]))
|
|
|
|
card_sections_rows: dict[str, list] = {}
|
|
for section_title, card_prefix in CARD_SECTIONS:
|
|
card_sections_rows[section_title] = []
|
|
for row in rows:
|
|
orig_path = row.get("_orig_rel_path", "")
|
|
for section_title, card_prefix in CARD_SECTIONS:
|
|
if orig_path.startswith(card_prefix + "/"):
|
|
card_sections_rows[section_title].append(row)
|
|
break
|
|
|
|
NIGHTLY_ROOT = REPO_ROOT / "tests" / "e2e" / "nightly"
|
|
WEEKLY_ROOT = REPO_ROOT / "tests" / "e2e" / "weekly"
|
|
|
|
def _scan_tests(root):
|
|
py_rows = []
|
|
yaml_rows = []
|
|
for fp in sorted(root.rglob("test_*.py")):
|
|
src = fp.read_text(encoding="utf-8", errors="replace")
|
|
py_rows.extend(_process_test_file(fp, src, root))
|
|
for fp in sorted(root.rglob("*.yaml")):
|
|
src = fp.read_text(encoding="utf-8", errors="replace")
|
|
yaml_rows.extend(_process_yaml_file(fp, src, root))
|
|
combined = py_rows + yaml_rows
|
|
combined.sort(key=lambda r: (r["Test file"], r["Test method"]))
|
|
return combined
|
|
|
|
nightly_rows = _scan_tests(NIGHTLY_ROOT)
|
|
weekly_rows = _scan_tests(WEEKLY_ROOT)
|
|
|
|
header = "| " + " | ".join(COLUMNS) + " |"
|
|
separator = "| " + " | ".join(["---"] * len(COLUMNS)) + " |"
|
|
|
|
output = "The coverage of e2e is as follows:\n\n"
|
|
for section_title, card_prefix in CARD_SECTIONS:
|
|
section_rows = card_sections_rows[section_title]
|
|
output += f"## {section_title}\n\n"
|
|
output += header + "\n"
|
|
output += separator + "\n"
|
|
for row in section_rows:
|
|
vals = [row.get(col, EMPTY) for col in COLUMNS]
|
|
output += "| " + " | ".join(vals) + " |\n"
|
|
output += "\n"
|
|
|
|
for section_title, rows_data in [("Nightly Tests", nightly_rows), ("Weekly Tests", weekly_rows)]:
|
|
output += f"## {section_title}\n\n"
|
|
output += header + "\n"
|
|
output += separator + "\n"
|
|
for row in rows_data:
|
|
vals = [row.get(col, EMPTY) for col in COLUMNS]
|
|
output += "| " + " | ".join(vals) + " |\n"
|
|
output += "\n"
|
|
|
|
print(output)
|
|
|
|
OUTPUT_FILE.write_text(output.rstrip("\n") + "\n", encoding="utf-8")
|
|
print(f"\nWritten to {OUTPUT_FILE}")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|