[router] allow router launch server to use grpc mode (#11600)
This commit is contained in:
@@ -1,4 +1,5 @@
|
|||||||
import argparse
|
import argparse
|
||||||
|
import asyncio
|
||||||
import copy
|
import copy
|
||||||
import logging
|
import logging
|
||||||
import multiprocessing as mp
|
import multiprocessing as mp
|
||||||
@@ -13,7 +14,6 @@ import requests
|
|||||||
from setproctitle import setproctitle
|
from setproctitle import setproctitle
|
||||||
from sglang_router.launch_router import RouterArgs, launch_router
|
from sglang_router.launch_router import RouterArgs, launch_router
|
||||||
|
|
||||||
from sglang.srt.entrypoints.http_server import launch_server
|
|
||||||
from sglang.srt.server_args import ServerArgs
|
from sglang.srt.server_args import ServerArgs
|
||||||
from sglang.srt.utils import is_port_available
|
from sglang.srt.utils import is_port_available
|
||||||
|
|
||||||
@@ -72,7 +72,15 @@ def run_server(server_args, dp_rank):
|
|||||||
# Set SGLANG_DP_RANK environment variable
|
# Set SGLANG_DP_RANK environment variable
|
||||||
os.environ["SGLANG_DP_RANK"] = str(dp_rank)
|
os.environ["SGLANG_DP_RANK"] = str(dp_rank)
|
||||||
|
|
||||||
launch_server(server_args)
|
# Launch server in appropriate mode (HTTP or gRPC)
|
||||||
|
if server_args.grpc_mode:
|
||||||
|
from sglang.srt.entrypoints.grpc_server import serve_grpc
|
||||||
|
|
||||||
|
asyncio.run(serve_grpc(server_args))
|
||||||
|
else:
|
||||||
|
from sglang.srt.entrypoints.http_server import launch_server
|
||||||
|
|
||||||
|
launch_server(server_args)
|
||||||
|
|
||||||
|
|
||||||
def launch_server_process(
|
def launch_server_process(
|
||||||
@@ -186,8 +194,10 @@ def main():
|
|||||||
)
|
)
|
||||||
|
|
||||||
# Update router args with worker URLs
|
# Update router args with worker URLs
|
||||||
|
# Use grpc:// protocol if server is in gRPC mode, otherwise http://
|
||||||
|
protocol = "grpc" if server_args.grpc_mode else "http"
|
||||||
router_args.worker_urls = [
|
router_args.worker_urls = [
|
||||||
f"http://{server_args.host}:{port}" for port in worker_ports
|
f"{protocol}://{server_args.host}:{port}" for port in worker_ports
|
||||||
]
|
]
|
||||||
|
|
||||||
# Start the router
|
# Start the router
|
||||||
|
|||||||
Reference in New Issue
Block a user