EngineX-Hygon/sglang
5
0
Fork 0
Code Issues Pull Requests Actions 7 Projects Releases Wiki Activity

Enable MLA by default (#1447)

Browse Source
This commit is contained in:
Ke Bao
2024-09-17 19:42:48 +08:00
committed by GitHub
parent 90a26be31c
commit c6b6d2e71b
8 changed files with 16 additions and 18 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
python/sglang/srt/model_executor/model_runner.py
View File

@@ -91,7 +91,7 @@ class ModelRunner:
"attention_backend": server_args.attention_backend,
"sampling_backend": server_args.sampling_backend,
"triton_attention_reduce_in_fp32": server_args.triton_attention_reduce_in_fp32,
"enable_mla": server_args.enable_mla,
"disable_mla": server_args.disable_mla,
"torchao_config": server_args.torchao_config,
}
)
@@ -329,7 +329,7 @@ class ModelRunner:
)
if (
self.model_config.attention_arch == AttentionArch.MLA
and self.server_args.enable_mla
and not self.server_args.disable_mla
):
cell_size = (
(self.model_config.kv_lora_rank + self.model_config.qk_rope_head_dim)
@@ -397,7 +397,7 @@ class ModelRunner:
)
if (
self.model_config.attention_arch == AttentionArch.MLA
and self.server_args.enable_mla
and not self.server_args.disable_mla
):
self.token_to_kv_pool = MLATokenToKVPool(
self.max_total_num_tokens,