Increase the number of thread limitation for tp worker managers. (#567)

benchmark/latency_throughput/bench_throughput.py

@@ -250,9 +250,14 @@ def main(args: argparse.Namespace):
     np.random.seed(args.seed)
 
     api_url = f"http://{args.host}:{args.port}/generate"
-    tokenizer = AutoTokenizer.from_pretrained(
-        args.tokenizer, trust_remote_code=args.trust_remote_code
-    )
+    if args.tokenizer.endswith(".json") or args.tokenizer.endswith(".model"):
+        from sglang.srt.hf_transformers_utils import get_tokenizer
+
+        tokenizer = get_tokenizer(args.tokenizer)
+    else:
+        tokenizer = AutoTokenizer.from_pretrained(
+            args.tokenizer, trust_remote_code=args.trust_remote_code
+        )
 
     if args.dataset:
         input_requests = sample_requests(args.dataset, args.num_prompts, tokenizer)
@@ -272,7 +277,7 @@ def main(args: argparse.Namespace):
         for i in range(args.num_prompts):
             prompt = tokenizer.decode(
                 [
-                    (offsets[i] + i + j) % tokenizer.vocab_size
+                    (offsets[i] + i + j) % (tokenizer.vocab_size - 129) + 128
                     for j in range(input_lens[i])
                 ]
             )

benchmark/latency_throughput/test_latency.py

@@ -8,6 +8,7 @@ if __name__ == "__main__":
     parser.add_argument("--host", type=str, default="http://127.0.0.1")
     parser.add_argument("--port", type=int, default=None)
     parser.add_argument("--backend", type=str, default="srt")
+    parser.add_argument("--batch-size", type=int, default=1)
     parser.add_argument("--max-tokens", type=int, default=256)
     args = parser.parse_args()
 
@@ -33,7 +34,7 @@ if __name__ == "__main__":
         response = requests.post(
             url + "/generate",
             json={
-                "text": prompt,
+                "text": [prompt] * args.batch_size,
                 "sampling_params": {
                     "temperature": 0,
                     "max_new_tokens": max_new_tokens,
@@ -90,5 +91,5 @@ if __name__ == "__main__":
         ret = response.json()
     print(ret)
 
-    speed = max_new_tokens / latency
+    speed = args.batch_size * max_new_tokens / latency
     print(f"latency: {latency:.2f} s, speed: {speed:.2f} token/s")