EngineX-Hygon/sglang
5
0
Fork 0
Code Issues Pull Requests Actions 7 Projects Releases Wiki Activity

Increase interpreter parallelism (#46)

Browse Source
This commit is contained in:
Lianmin Zheng
2024-01-18 13:30:10 -08:00
committed by GitHub
parent 05b4c398df
commit 2b079f8931
1 changed files with 1 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
python/sglang/lang/interpreter.py
View File

@@ -80,7 +80,7 @@ def run_program_batch(
# Run all programs # Run all programs
if num_threads == "auto": if num_threads == "auto":
num_threads = multiprocessing.cpu_count() num_threads = min(64, multiprocessing.cpu_count() * 8)
num_threads = min(num_threads, len(batch_arguments)) num_threads = min(num_threads, len(batch_arguments))
if num_threads == 1: if num_threads == 1: