44a8301424
### What this PR does / why we need it? Adapt Disaggregated Prefill feature onto Ascend device ### Does this PR introduce _any_ user-facing change? no ### How was this patch tested? The test usage has been provided alongwith the PR, in examples/offline_disaggregated_prefill_npu.py To run it, do this ``` export PROMPT_DEVICE_ID=0,1 export DECODE_DEVICE_ID=2,3 python examples/offline_disaggregated_prefill_npu.py ``` --------- Signed-off-by: ZihuiQian <qianzihui@huawei.com> Co-authored-by: ZihuiQian <qianzihui@huawei.com>
44 lines
1.4 KiB
Python
44 lines
1.4 KiB
Python
import os
|
|
from typing import Any, Callable, Dict
|
|
|
|
env_variables: Dict[str, Callable[[], Any]] = {
|
|
# max compile thread num
|
|
"MAX_JOBS":
|
|
lambda: os.getenv("MAX_JOBS", None),
|
|
"CMAKE_BUILD_TYPE":
|
|
lambda: os.getenv("CMAKE_BUILD_TYPE"),
|
|
"COMPILE_CUSTOM_KERNELS":
|
|
lambda: bool(int(os.getenv("COMPILE_CUSTOM_KERNELS", "1"))),
|
|
"SOC_VERSION":
|
|
lambda: os.getenv("SOC_VERSION", "ASCEND910B1"),
|
|
# If set, vllm-ascend will print verbose logs during compilation
|
|
"VERBOSE":
|
|
lambda: bool(int(os.getenv('VERBOSE', '0'))),
|
|
"ASCEND_HOME_PATH":
|
|
lambda: os.getenv("ASCEND_HOME_PATH", None),
|
|
"LD_LIBRARY_PATH":
|
|
lambda: os.getenv("LD_LIBRARY_PATH", None),
|
|
# Used for disaggregated prefilling
|
|
"HCCN_PATH":
|
|
lambda: os.getenv("HCCN_PATH", "/usr/local/Ascend/driver/tools/hccn_tool"),
|
|
"PROMPT_DEVICE_ID":
|
|
lambda: os.getenv("PROMPT_DEVICE_ID", None),
|
|
"DECODE_DEVICE_ID":
|
|
lambda: os.getenv("DECODE_DEVICE_ID", None),
|
|
"LLMDATADIST_COMM_PORT":
|
|
lambda: os.getenv("LLMDATADIST_COMM_PORT", "26000"),
|
|
"LLMDATADIST_SYNC_CACHE_WAIT_TIME":
|
|
lambda: os.getenv("LLMDATADIST_SYNC_CACHE_WAIT_TIME", "5000")
|
|
}
|
|
|
|
|
|
def __getattr__(name: str):
|
|
# lazy evaluation of environment variables
|
|
if name in env_variables:
|
|
return env_variables[name]()
|
|
raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
|
|
|
|
|
|
def __dir__():
|
|
return list(env_variables.keys())
|