[main][refactor] Refactoring forward_context and model_runner_v1 (#1979)
### What this PR does / why we need it?
A refactoring of forward_context and model_runner_v1, add some context
which is necessary in model inference into forward_context, and refactor
dummy_run logic, make it more reasonable.
Some details for this PR:
Add `ascend_forward_context`;
Update mc2_v2 op, and support `active_mask` param;
Update scripts in examples dir;
refactor `dummy_run` logic;
Add soc_version for A2 and A3;
### Does this PR introduce _any_ user-facing change?
No change at user-facing.
### How was this patch tested?
- vLLM version: v0.10.0
- vLLM main:
57c22e57f9
Signed-off-by: zzzzwwjj <1183291235@qq.com>
This commit is contained in:
@@ -40,9 +40,10 @@ from vllm.v1.worker.worker_base import WorkerBase
|
||||
|
||||
from vllm_ascend.ascend_config import init_ascend_config
|
||||
from vllm_ascend.device_allocator.camem import CaMemAllocator
|
||||
from vllm_ascend.distributed.parallel_state import init_ascend_model_parallel
|
||||
from vllm_ascend.platform import NPUPlatform
|
||||
from vllm_ascend.utils import (sleep_mode_enabled, try_register_lib,
|
||||
vllm_version_is)
|
||||
from vllm_ascend.utils import (init_ascend_soc_version, sleep_mode_enabled,
|
||||
try_register_lib, vllm_version_is)
|
||||
from vllm_ascend.worker.model_runner_v1 import NPUModelRunner
|
||||
|
||||
if not vllm_version_is("0.10.0"):
|
||||
@@ -134,6 +135,7 @@ class NPUWorker(WorkerBase):
|
||||
NPUPlatform.empty_cache()
|
||||
self.init_npu_memory = NPUPlatform.mem_get_info()[0]
|
||||
|
||||
init_ascend_soc_version()
|
||||
# Initialize the distributed environment.
|
||||
self._init_worker_distributed_environment()
|
||||
# Set random seed.
|
||||
@@ -272,20 +274,8 @@ class NPUWorker(WorkerBase):
|
||||
def pin_lora(self, lora_id: int) -> bool:
|
||||
return self.model_runner.pin_lora(lora_id)
|
||||
|
||||
def _get_max_num_tokens_and_with_prefill(self):
|
||||
max_num_tokens = 1
|
||||
with_prefill = False
|
||||
if self.model_runner.dp_size > 1:
|
||||
max_num_tokens, with_prefill = self.model_runner._get_forward_metadata_across_dp(
|
||||
max_num_tokens, with_prefill)
|
||||
return max_num_tokens, with_prefill
|
||||
|
||||
def execute_dummy_batch(self) -> None:
|
||||
max_num_tokens, with_prefill = self._get_max_num_tokens_and_with_prefill(
|
||||
)
|
||||
self.model_runner._dummy_run(max_num_tokens,
|
||||
is_compile=False,
|
||||
with_prefill=with_prefill)
|
||||
self.model_runner._dummy_run(1)
|
||||
|
||||
def _init_worker_distributed_environment(self) -> None:
|
||||
"""Initialize the distributed environment."""
|
||||
@@ -295,6 +285,7 @@ class NPUWorker(WorkerBase):
|
||||
ensure_model_parallel_initialized(
|
||||
self.parallel_config.tensor_parallel_size,
|
||||
self.parallel_config.pipeline_parallel_size)
|
||||
init_ascend_model_parallel(self.parallel_config)
|
||||
ensure_kv_transfer_initialized(self.vllm_config)
|
||||
|
||||
def _init_profiler(self):
|
||||
|
||||
Reference in New Issue
Block a user