diff --git a/examples/disaggregated_prefill_v1/load_balance_proxy_layerwise_server_example.py b/examples/disaggregated_prefill_v1/load_balance_proxy_layerwise_server_example.py index 8bbc359..53353ce 100644 --- a/examples/disaggregated_prefill_v1/load_balance_proxy_layerwise_server_example.py +++ b/examples/disaggregated_prefill_v1/load_balance_proxy_layerwise_server_example.py @@ -88,6 +88,7 @@ import argparse import asyncio import functools import heapq +import ipaddress import os import sys import threading @@ -116,6 +117,12 @@ class ServerState: self.host = host self.port = port self.url = f'http://{host}:{port}/v1' + try: + ip = ipaddress.ip_address(self.host) + if isinstance(ip, ipaddress.IPv6Address): + self.url = f'http://[{host}]:{port}/v1' + except Exception: + pass self.client = httpx.AsyncClient(timeout=None, base_url=self.url, limits=httpx.Limits( @@ -356,6 +363,9 @@ async def send_request_to_service(client: httpx.AsyncClient, req_data = req_data.copy() req_data["stream"] = False req_data["max_tokens"] = 1 + req_data["min_tokens"] = 1 + if "max_completion_tokens" in req_data: + req_data["max_completion_tokens"] = 1 if "stream_options" in req_data: del req_data["stream_options"] headers = { diff --git a/examples/disaggregated_prefill_v1/load_balance_proxy_server_example.py b/examples/disaggregated_prefill_v1/load_balance_proxy_server_example.py index 0694ace..c02a6fa 100644 --- a/examples/disaggregated_prefill_v1/load_balance_proxy_server_example.py +++ b/examples/disaggregated_prefill_v1/load_balance_proxy_server_example.py @@ -88,6 +88,7 @@ import argparse import asyncio import functools import heapq +import ipaddress import json import os import sys @@ -118,6 +119,12 @@ class ServerState: self.host = host self.port = port self.url = f'http://{host}:{port}/v1' + try: + ip = ipaddress.ip_address(self.host) + if isinstance(ip, ipaddress.IPv6Address): + self.url = f'http://[{host}]:{port}/v1' + except Exception: + pass self.client = httpx.AsyncClient(timeout=None, base_url=self.url, limits=httpx.Limits( @@ -366,6 +373,8 @@ async def send_request_to_service(client: httpx.AsyncClient, req_data["stream"] = False req_data["max_tokens"] = 1 req_data["min_tokens"] = 1 + if "max_completion_tokens" in req_data: + req_data["max_completion_tokens"] = 1 if "stream_options" in req_data: del req_data["stream_options"] headers = { diff --git a/vllm_ascend/distributed/mooncake/transfer_engine.py b/vllm_ascend/distributed/mooncake/transfer_engine.py index e515da6..d4e172b 100644 --- a/vllm_ascend/distributed/mooncake/transfer_engine.py +++ b/vllm_ascend/distributed/mooncake/transfer_engine.py @@ -1,3 +1,4 @@ +import ipaddress import threading from typing import Optional @@ -8,6 +9,15 @@ _global_te_lock = threading.Lock() def get_global_te(hostname: str, device_name: Optional[str]): + try: + ip = ipaddress.ip_address(hostname) + if isinstance(ip, ipaddress.IPv6Address): + raise RuntimeError( + "The backend of mooncake's Ascend Direct Xfer Library currently does not support IPv6." + ) + except ValueError: + pass + global _global_te if _global_te is None: with _global_te_lock: diff --git a/vllm_ascend/envs.py b/vllm_ascend/envs.py index db149ac..72a7792 100644 --- a/vllm_ascend/envs.py +++ b/vllm_ascend/envs.py @@ -162,11 +162,6 @@ env_variables: Dict[str, Callable[[], Any]] = { # Whether to enable msMonitor tool to monitor the performance of vllm-ascend. "MSMONITOR_USE_DAEMON": lambda: bool(int(os.getenv("MSMONITOR_USE_DAEMON", '0'))), - # Timeout (in seconds) for delayed KVCache block release. In the prefill - # node, if a request is marked for delayed KV block release and the blocks - # are not freed within this timeout, they will be forcibly released. - "VLLM_ASCEND_KVCACHE_DELAY_FREE_TIMEOUT": - lambda: int(os.getenv("VLLM_ASCEND_KVCACHE_DELAY_FREE_TIMEOUT", 250)), "VLLM_ASCEND_ENABLE_MLAPO": lambda: bool(int(os.getenv("VLLM_ASCEND_ENABLE_MLAPO", '0'))), # Whether to enable transpose weight and cast format to FRACTAL_NZ.