From 2b079f89315553897d9abe49366a24534eee7ec0 Mon Sep 17 00:00:00 2001 From: Lianmin Zheng Date: Thu, 18 Jan 2024 13:30:10 -0800 Subject: [PATCH] Increase interpreter parallelism (#46) --- python/sglang/lang/interpreter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/sglang/lang/interpreter.py b/python/sglang/lang/interpreter.py index 150fd04fc..a7fffa508 100644 --- a/python/sglang/lang/interpreter.py +++ b/python/sglang/lang/interpreter.py @@ -80,7 +80,7 @@ def run_program_batch( # Run all programs if num_threads == "auto": - num_threads = multiprocessing.cpu_count() + num_threads = min(64, multiprocessing.cpu_count() * 8) num_threads = min(num_threads, len(batch_arguments)) if num_threads == 1: