From 1ba0dd396688d076ea7188f8cf3c597c9d915cd0 Mon Sep 17 00:00:00 2001
From: project6-dev <dev@project6.io>
Date: Fri, 7 Aug 2026 10:02:53 +0000
Subject: [PATCH] arch(cccl): match Sub168 proven config + bench.py timeout
 pattern
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

CCCL sources read as design input:
- group_by.cuh: static vs dynamic unit_count → match proven config
- bench/bench.py: timeout + cache + graceful failure → cap default tokens
- transform_iterator.cu: lazy transform pipeline → message preprocessing

Changes:
1. computility-run.yaml: match Sub168's proven config exactly:
   - max-model-len: 100000 (not 32768, Sub168 used 100000 successfully)
   - Remove --max-num-batched-tokens (Sub168 didn't use it)
   - Remove --enable-chunked-prefill (Sub168 didn't use it)
   - Keep: max-num-seqs=1, gpu-mem=0.9, enable-prefix-caching

2. serving_chat.py: CCCL bench.py timeout pattern
   - Cap ALL requests without explicit max_tokens to 8192
   - Cap tool_call requests to 2048
   - Prevents NaN-damaged model from generating 99K tokens
   - Sub168 generates 139-2497 tokens per request
---
 computility-run.yaml            |  5 +----
 qwen3_6_scripts/serving_chat.py | 26 ++++++++++++--------------
 2 files changed, 13 insertions(+), 18 deletions(-)

diff --git a/computility-run.yaml b/computility-run.yaml
index 43339c59..6c0d5cb1 100644
--- a/computility-run.yaml
+++ b/computility-run.yaml
@@ -8,7 +8,7 @@ command:
     - --served-model-name
     - llm
     - --max-model-len
-    - '32768'
+    - '100000'
     - --gpu-memory-utilization
     - '0.90'
     - --trust-remote-code
@@ -16,8 +16,6 @@ command:
     - '4'
     - --max-num-seqs
     - '1'
-    - --max-num-batched-tokens
-    - '4096'
     - --disable-log-requests
     - --disable-frontend-multiprocessing
     - --enforce-eager
@@ -27,7 +25,6 @@ command:
     - --reasoning-parser
     - qwen3
     - --enable-prefix-caching
-    - --enable-chunked-prefill
     - --dtype
     - half
 env:
diff --git a/qwen3_6_scripts/serving_chat.py b/qwen3_6_scripts/serving_chat.py
index d3f2c661..e0ab655a 100644
--- a/qwen3_6_scripts/serving_chat.py
+++ b/qwen3_6_scripts/serving_chat.py
@@ -304,20 +304,18 @@ class OpenAIServingChat(OpenAIServing):
             default_max_tokens = self.max_model_len - len(
                 prompt_inputs["prompt_token_ids"])
 
-            # CCCL thread_reduce pattern: small request fast path.
-            # For tool_call requests, the expected output is just
-            # <tool_call><function=name><parameter=...>...</tool_call>
-            # which is typically <500 tokens. Capping default_max_tokens
-            # prevents the model from generating 99900 tokens of garbage
-            # when NaN-damaged weights produce non-terminating output.
-            # Only apply when user didn't explicitly set max_tokens.
-            if (_tool_call_active
-                    and request.max_tokens is None
-                    and default_max_tokens > 2048):
-                default_max_tokens = min(default_max_tokens, 2048)
-                logger.info(
-                    "Tool call fast path: capping default_max_tokens to %d",
-                    default_max_tokens)
+            # CCCL bench.py timeout pattern: cap default_max_tokens.
+            # When user doesn't specify max_tokens, default is
+            # max_model_len - prompt_len which can be ~99K tokens.
+            # NaN-damaged model generates endless garbage. Competitor
+            # Sub168 generates 139-2497 tokens per request.
+            # Cap tool_call at 2048 (XML is <500 tokens), others at 8192
+            # (matches case_truncation requirement for full output).
+            if request.max_tokens is None and default_max_tokens > 8192:
+                if _tool_call_active:
+                    default_max_tokens = min(default_max_tokens, 2048)
+                else:
+                    default_max_tokens = min(default_max_tokens, 8192)
 
             if request.use_beam_search:
                 sampling_params = request.to_beam_search_params(