Increase interpreter parallelism (#46)

2024-01-18 13:30:10 -08:00
parent 05b4c398df
commit 2b079f8931
1 changed files with 1 additions and 1 deletions
--- a/python/sglang/lang/interpreter.py
+++ b/python/sglang/lang/interpreter.py
@@ -80,7 +80,7 @@ def run_program_batch(

    # Run all programs
    if num_threads == "auto":
-        num_threads = multiprocessing.cpu_count()
+        num_threads = min(64, multiprocessing.cpu_count() * 8)
    num_threads = min(num_threads, len(batch_arguments))

    if num_threads == 1: