### What this PR does / why we need it?
Fix #3891.
The empty of `moe_comm_method` in the above issue is due to the wrong
check for MoE models. To be specific, the method `is_moe_model` only
checks whether a text-only model is a MoE model, without considering
multi-modal models, e.g., `VL` and `Omni`.
Check the config dict recursively to find if it has a key contains
"expert", without checking the model architecture.
It is worth noting that, we can't verify a model by if it contains
`FusedMoE` module because `is_moe_model` is called somewhere before the
model loading, e.g., it's called when updating the ACLGraph config in
platform initialization.
- vLLM version: v0.11.0
- vLLM main:
83f478bb19
---------
Signed-off-by: shen-shanshan <467638484@qq.com>
This commit is contained in:
@@ -624,14 +624,24 @@ def shared_expert_dp_enabled() -> bool:
|
||||
|
||||
|
||||
def is_moe_model(vllm_config: VllmConfig):
|
||||
"""Checks if the model is a MoE model by config"""
|
||||
global _IS_MOE_MODEL
|
||||
if _IS_MOE_MODEL is None:
|
||||
config = vllm_config.model_config.hf_config
|
||||
_IS_MOE_MODEL = any('experts' in key.lower()
|
||||
for key in config.to_dict())
|
||||
model_configs = vllm_config.model_config.hf_config.to_dict()
|
||||
_IS_MOE_MODEL = _is_contain_expert(model_configs)
|
||||
return _IS_MOE_MODEL
|
||||
|
||||
|
||||
def _is_contain_expert(config: Any):
|
||||
if isinstance(config, dict):
|
||||
for k, v in config.items():
|
||||
if "expert" in str(k):
|
||||
return True
|
||||
if _is_contain_expert(v):
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def weak_ref_tensor(tensor: Any) -> Any:
|
||||
"""
|
||||
Create a weak reference to a tensor.
|
||||
|
||||
Reference in New Issue
Block a user