From 57a2216143ad9766ecc064063543ce830b92e80e Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 7 Aug 2026 08:18:41 +0000 Subject: [PATCH] fix: max_num_seqs=2 for n=2 support + remove protocol n clamp MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Sub168 (competitor) passes t2_n_2 with n=2 at 1.50s even with max_num_seqs likely >1. Our max_num_seqs=1 made n=2 crash. Changes: - computility-run.yaml: max-num-seqs 1→2 (200GB total VRAM sufficient) - protocol.py: remove n>1 clamp, let serving_chat scheduler guard handle it - serving_chat.py retains try/except guard for get_scheduler_config Risk: if 2 concurrent seqs OOM, service crashes. But concurrency=1 means only 1 request at a time, so n=2 just generates 2 answers sequentially. CCCL input: tuning_topk.cuh (bits_per_pass=11 for float32, threads=512), tuning_transform.cuh (cc_to_min_bytes_in_flight: B200=64KB, A100=16KB, BI-V100 should use 48-64KB based on per-SM BW=56GB/s) --- computility-run.yaml | 2 +- qwen3_6_scripts/protocol.py | 10 ++-------- 2 files changed, 3 insertions(+), 9 deletions(-) diff --git a/computility-run.yaml b/computility-run.yaml index 6c0d5cb1..e5da63a7 100644 --- a/computility-run.yaml +++ b/computility-run.yaml @@ -15,7 +15,7 @@ command: - -tp - '4' - --max-num-seqs - - '1' + - '2' - --disable-log-requests - --disable-frontend-multiprocessing - --enforce-eager diff --git a/qwen3_6_scripts/protocol.py b/qwen3_6_scripts/protocol.py index 3dcd7dab..486de91f 100644 --- a/qwen3_6_scripts/protocol.py +++ b/qwen3_6_scripts/protocol.py @@ -418,14 +418,8 @@ class ChatCompletionRequest(OpenAIBaseModel): if data.get("max_completion_tokens") is not None and data.get("max_tokens") is None: data["max_tokens"] = data["max_completion_tokens"] - # Clamp n to 1 to prevent engine crash. Competition config uses - # max_num_seqs=1; n>1 deadlocks the scheduler (break-not-continue bug) - # or causes OOM, crashing the engine for ALL subsequent requests. - # Sub508: t2_n_2 → HTTP 500 → 19 cascade failures. - # t2_n_2 will FAIL (1 choice instead of 2) but prevents cascade. - n_val = data.get("n") - if n_val is not None and isinstance(n_val, int) and n_val > 1: - data["n"] = 1 + # n > max_num_seqs: clamp handled in serving_chat.py via scheduler check. + # With max_num_seqs=2, n=2 should work. n>2 will be clamped there. # Map thinking parameter → chat_template_kwargs.enable_thinking # OpenAI API format: thinking={"type":"enabled"} / {"type":"disabled"}