fix(protocol): clamp max_tokens to available context — fix t3_max_tokens_max HTTP 400
When max_tokens >= max_model_len, vLLM engine rejects the request. Clamp to (max_model_len - prompt_tokens) in both to_sampling_params and to_beam_search_params so oversized max_tokens values degrade gracefully instead of returning HTTP 400. CCCL logical.cu pattern: handle boundary conditions (empty range, overflow) gracefully instead of hard-failing.
This commit is contained in:
@@ -305,6 +305,8 @@ class ChatCompletionRequest(OpenAIBaseModel):
|
|||||||
max_tokens = self.max_tokens
|
max_tokens = self.max_tokens
|
||||||
if max_tokens is None:
|
if max_tokens is None:
|
||||||
max_tokens = default_max_tokens
|
max_tokens = default_max_tokens
|
||||||
|
if default_max_tokens > 0:
|
||||||
|
max_tokens = min(max_tokens, default_max_tokens)
|
||||||
|
|
||||||
n = self.n if self.n is not None else 1
|
n = self.n if self.n is not None else 1
|
||||||
temperature = self.temperature if self.temperature is not None else 0.0
|
temperature = self.temperature if self.temperature is not None else 0.0
|
||||||
@@ -321,6 +323,10 @@ class ChatCompletionRequest(OpenAIBaseModel):
|
|||||||
max_tokens = self.max_tokens
|
max_tokens = self.max_tokens
|
||||||
if max_tokens is None:
|
if max_tokens is None:
|
||||||
max_tokens = default_max_tokens
|
max_tokens = default_max_tokens
|
||||||
|
# Clamp to available context space so requests with max_tokens ≥
|
||||||
|
# max_model_len don't get rejected with HTTP 400.
|
||||||
|
if default_max_tokens > 0:
|
||||||
|
max_tokens = min(max_tokens, default_max_tokens)
|
||||||
|
|
||||||
prompt_logprobs = self.prompt_logprobs
|
prompt_logprobs = self.prompt_logprobs
|
||||||
if prompt_logprobs is None and self.echo:
|
if prompt_logprobs is None and self.echo:
|
||||||
|
|||||||
Reference in New Issue
Block a user