fix: topk_softmax .so + fp32 router + enforce_eager + comp168 params

This commit is contained in:
project6-dev
2026-08-12 11:14:01 +00:00
parent a72877a509
commit 07e8681e2e
12 changed files with 1631 additions and 13 deletions

View File

@@ -394,6 +394,9 @@ class OpenAIServingChat(OpenAIServing):
assert prompt_inputs is not None
sampling_params: Union[SamplingParams, BeamSearchParams]
# OpenAI API: max_completion_tokens takes precedence over max_tokens
if request.max_completion_tokens is not None and request.max_tokens is None:
request.max_tokens = request.max_completion_tokens
default_max_tokens = self.max_model_len - len(
prompt_inputs["prompt_token_ids"])
if request.use_beam_search: