EngineX-Hygon/sglang
5
0
Fork 0
Code Issues Pull Requests Actions 7 Projects Releases Wiki Activity

[Auto Sync] Update model_runner.py (20250920) (#10679)

Browse Source 
Co-authored-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>
This commit is contained in:
Yineng Zhang
2025-09-19 18:26:54 -07:00
committed by GitHub
parent d88ef4a388
commit 60e2a7cead
1 changed files with 20 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files

31
python/sglang/srt/model_executor/model_runner.py
View File

@@ -147,6 +147,25 @@ from sglang.srt.weight_sync.tensor_bucket import (
FlattenedTensorMetadata, FlattenedTensorMetadata,
) )
MLA_ATTENTION_BACKENDS = [
"aiter",
"flashinfer",
"fa3",
"fa4",
"triton",
"flashmla",
"cutlass_mla",
"trtllm_mla",
"ascend",
]
def add_mla_attention_backend(backend_name):
if backend_name not in MLA_ATTENTION_BACKENDS:
MLA_ATTENTION_BACKENDS.append(backend_name)
logger.info(f"Added {backend_name} to MLA_ATTENTION_BACKENDS.")
_is_hip = is_hip() _is_hip = is_hip()
_is_npu = is_npu() _is_npu = is_npu()
_is_cpu_amx_available = cpu_has_amx_support() _is_cpu_amx_available = cpu_has_amx_support()
@@ -513,17 +532,7 @@ class ModelRunner:
) )
elif self.use_mla_backend: elif self.use_mla_backend:
if server_args.device != "cpu": if server_args.device != "cpu":
if server_args.attention_backend in [ if server_args.attention_backend in MLA_ATTENTION_BACKENDS:
"aiter",
"flashinfer",
"fa3",
"fa4",
"triton",
"flashmla",
"cutlass_mla",
"trtllm_mla",
"ascend",
]:
logger.info( logger.info(
f"MLA optimization is turned on. Use {server_args.attention_backend} backend." f"MLA optimization is turned on. Use {server_args.attention_backend} backend."
) )