dylanyunlong/project_6
1
0
Fork 0
Code Issues Pull Requests Actions Projects Releases Wiki Activity

Revert "feat: 3-tier ixformer flash prefill dispatch + OpenCompass max_tokens clamp + n>1 fanout + index sanitizer"

Browse Source 
This reverts commit cdec569977.
This commit is contained in:
Claude
2026-08-16 15:42:36 +00:00
parent 7a7ddf38db
commit 5172f94b1f
3 changed files with 10 additions and 274 deletions
Download Patch File Download Diff File Expand all files Collapse all files

33
qwen3_6_scripts/serving_chat.py
View File

@@ -118,17 +118,12 @@ def _sequential_greedy_fanout_count(
request: ChatCompletionRequest,
max_num_seqs: int,
) -> int:
"""Return the supported fan-out width, or zero.
When max_num_seqs=1 (competition fixed config), vLLM cannot schedule
n>1 natively. We sequentially execute n independent n=1 requests and
merge them. This works for any temperature — deterministic (temp=0)
produces identical choices, stochastic produces diverse ones.
"""
"""Return the supported deterministic fan-out width, or zero."""
n = request.n if request.n is not None else 1
if (
max_num_seqs == 1
and 2 <= n <= 4
and n == 2
and request.temperature == 0
and not request.stream
and not request.use_beam_search
and request.best_of is None
@@ -143,8 +138,8 @@ def _merge_sequential_chat_responses(
request_id: str,
created_time: int,
) -> ChatCompletionResponse:
if len(responses) < 2:
raise ValueError("fan-out requires at least two responses")
if len(responses) != 2:
raise ValueError("deterministic fan-out requires exactly two responses")
first = responses[0]
if any(response.model != first.model for response in responses):
@@ -402,16 +397,8 @@ class OpenAIServingChat(OpenAIServing):
# OpenAI API: max_completion_tokens takes precedence over max_tokens
if request.max_completion_tokens is not None and request.max_tokens is None:
request.max_tokens = request.max_completion_tokens
prompt_len = len(prompt_inputs["prompt_token_ids"])
default_max_tokens = self.max_model_len - prompt_len
# Clamp max_tokens so prompt + completion <= max_model_len.
# Without this, evaluation systems (e.g. OpenCompass) that send
# max_tokens=131072 get 400 errors when prompt+max_tokens exceeds
# max_model_len, resulting in 0 score on all academic benchmarks.
if default_max_tokens < 1:
default_max_tokens = 1
if request.max_tokens is not None and request.max_tokens > default_max_tokens:
request.max_tokens = default_max_tokens
default_max_tokens = self.max_model_len - len(
prompt_inputs["prompt_token_ids"])
if request.use_beam_search:
sampling_params = request.to_beam_search_params(
default_max_tokens)
@@ -505,7 +492,7 @@ class OpenAIServingChat(OpenAIServing):
logger.error(
"Sequential greedy fan-out unexpectedly returned a stream")
return self.create_error_response(
f"Failed to aggregate n={fanout_count} completion")
"Failed to aggregate deterministic n=2 completion")
responses.append(child_response)
try:
@@ -516,11 +503,11 @@ class OpenAIServingChat(OpenAIServing):
)
except ValueError as error:
logger.error(
"Sequential fan-out aggregation failed: %s",
"Sequential greedy fan-out aggregation failed: %s",
type(error).__name__,
)
return self.create_error_response(
f"Failed to aggregate n={fanout_count} completion")
"Failed to aggregate deterministic n=2 completion")
if raw_request is not None:
metadata = RequestResponseMetadata(