EngineX-Hygon/sglang
5
0
Fork 0
Code Issues Pull Requests Actions 7 Projects Releases Wiki Activity

[Minor] improve code style

Browse Source
This commit is contained in:
Lianmin Zheng
2024-06-03 18:11:34 -07:00
parent 9f009261f2
commit 3bc01ac137
4 changed files with 67 additions and 17 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
benchmark/latency_throughput/bench_throughput.py
View File

@@ -149,12 +149,12 @@ async def send_request(
"inputs": prompt,
"parameters": params,
}
elif backend == "xinfer":
elif backend == "ginfer":
pass
else:
raise ValueError(f"Unknown backend: {backend}")
if backend != "xinfer":
if backend != "ginfer":
timeout = aiohttp.ClientTimeout(total=3 * 3600)
async with aiohttp.ClientSession(timeout=timeout) as session:
while True:
@@ -172,7 +172,7 @@ async def send_request(
print(output)
else:
import grpc
from xlm.proto import sampler_pb2, sampler_pb2_grpc
from ginfer import sampler_pb2, sampler_pb2_grpc
api_url = api_url.replace("http://", "").replace("/generate", "")
sampler_channel = grpc.aio.insecure_channel(api_url)
@@ -283,7 +283,7 @@ if __name__ == "__main__":
"--backend",
type=str,
default="srt",
choices=["vllm", "tgi", "srt", "lightllm", "xinfer"],
choices=["vllm", "tgi", "srt", "lightllm", "ginfer"],
)
parser.add_argument("--host", type=str, default="localhost")
parser.add_argument("--port", type=int, default=30000)

6
benchmark/latency_throughput/test_latency.py
View File

@@ -18,7 +18,7 @@ if __name__ == "__main__":
args.port = 21000
elif args.backend == "lightllm":
args.port = 22000
elif args.backend == "xinfer":
elif args.backend == "ginfer":
args.port = 9988
else:
raise ValueError(f"Invalid backend: {args.backend}")
@@ -60,9 +60,9 @@ if __name__ == "__main__":
"max_tokens": max_new_tokens,
},
)
elif args.backend == "xinfer":
elif args.backend == "ginfer":
import grpc
from xlm.proto import sampler_pb2, sampler_pb2_grpc
from ginfer import sampler_pb2, sampler_pb2_grpc
sampler_channel = grpc.insecure_channel(url.replace("http://", ""))
sampler = sampler_pb2_grpc.SamplerStub(sampler_channel)