[Bugfix] Fix the Eagle3 inference failure issue. (#4721)
### What this PR does / why we need it? Fix the Eagle3 inference failure issue. error message: "EngineCore encountered an issue. See stack trace (above) for the root cause." Fixes https://github.com/vllm-project/vllm-ascend/issues/4323 ### How was this patch tested? `vllm serve /nfs/1_AscendPackage/05_weights_public/Qwen3-32B \ --served-model-name Qwen3-32B \ -tp 4 \ --host "0.0.0.0" \ --port "8000" \ --trust-remote-code \ --speculative-config '{"method":"eagle3","model":"/home/scd/qwen3_32b_eagle3/","num_speculative_tokens":4,"draft_tensor_parallel_size":1}' \ --max-num-batched-tokens 4096 \ --max-model-len 4096` ``` curl http://localhost:8000/v1/completions \ -H "Content-Type: application/json" \ -d '{ "model": "Qwen3-32B", "prompt": "hi, where is the capital of France?", "max_tokens": 10, "temperature": 0 }' | python3 -m json.tool ``` vLLM version: v0.11.0 vLLM-ascend version: v0.11.0rc2 Signed-off-by: 17764591921 <sunchend@outlook.com>
This commit is contained in:
@@ -61,6 +61,7 @@ _IS_VL_MODEL = None
|
||||
_ENABLE_SP = None
|
||||
_HAS_LAYER_IDX = None
|
||||
_ENABLE_NZ = None
|
||||
_IS_EAGLE_MODE = None
|
||||
|
||||
|
||||
def is_310p():
|
||||
@@ -73,14 +74,20 @@ def is_310p():
|
||||
|
||||
def is_enable_nz(dtype: Optional[torch.dtype] = torch.int8,
|
||||
vllm_config: Optional[VllmConfig] = None) -> bool:
|
||||
global _ENABLE_NZ
|
||||
global _ENABLE_NZ, _IS_EAGLE_MODE
|
||||
if _ENABLE_NZ is None:
|
||||
if not vllm_config:
|
||||
raise ValueError(
|
||||
"vllm_config must be provided when _ENABLE_NZ is None")
|
||||
_ENABLE_NZ = envs_ascend.VLLM_ASCEND_ENABLE_NZ and vllm_config.model_config.hf_config.model_type != "qwen3_next"
|
||||
|
||||
_IS_EAGLE_MODE = (
|
||||
vllm_config.speculative_config is not None and
|
||||
getattr(vllm_config.speculative_config, 'method', None) in ("eagle", "eagle3")
|
||||
)
|
||||
|
||||
if dtype in [torch.float16, torch.bfloat16]:
|
||||
return False
|
||||
return _ENABLE_NZ if _IS_EAGLE_MODE else False
|
||||
return _ENABLE_NZ
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user