diff --git a/computility-run.yaml b/computility-run.yaml index 92dd03b0..56695b53 100644 --- a/computility-run.yaml +++ b/computility-run.yaml @@ -9,9 +9,9 @@ command: cp /workspace/qwen3_6_scripts/serving_chat.py "${VLLM_ROOT}/entrypoints/openai/serving_chat.py" 2>/dev/null && echo '[runtime] serving_chat.py redeployed' || echo '[runtime] serving_chat.py redeploy skipped'; cp /workspace/qwen3_6_scripts/multimodal_utils.py "${VLLM_ROOT}/multimodal/utils.py" 2>/dev/null && echo '[runtime] multimodal_utils.py redeployed' || echo '[runtime] multimodal_utils.py redeploy skipped'; cp /workspace/qwen3_6_scripts/envs.py "${VLLM_ROOT}/envs.py" 2>/dev/null && echo '[runtime] envs.py redeployed' || echo '[runtime] envs.py redeploy skipped'; - find "${VLLM_ROOT}" -name '*.pyc' -path '*/openai/*' -delete 2>/dev/null; - find "${VLLM_ROOT}" -name '*.pyc' -path '*/multimodal/*' -delete 2>/dev/null; - find "${VLLM_ROOT}" -maxdepth 1 -name 'envs*.pyc' -delete 2>/dev/null; + cp /workspace/qwen3_6_scripts/vendor_overrides/vllm/sampling_params.py "${VLLM_ROOT}/sampling_params.py" 2>/dev/null && echo '[runtime] sampling_params.py redeployed' || echo '[runtime] sampling_params.py redeploy skipped'; + find "${VLLM_ROOT}" -name '*.pyc' -delete 2>/dev/null; + find "${VLLM_ROOT}" -name '__pycache__' -type d -exec rm -rf {} + 2>/dev/null; exec python3 -m vllm.entrypoints.openai.api_server --model /model --served-model-name llm diff --git a/qwen3_6_scripts/serving_chat.py b/qwen3_6_scripts/serving_chat.py index 8b5ceeab..27d9e0aa 100644 --- a/qwen3_6_scripts/serving_chat.py +++ b/qwen3_6_scripts/serving_chat.py @@ -396,8 +396,23 @@ class OpenAIServingChat(OpenAIServing): # OpenAI API: max_completion_tokens takes precedence over max_tokens if request.max_completion_tokens is not None and request.max_tokens is None: request.max_tokens = request.max_completion_tokens - default_max_tokens = min(self.max_model_len - len( - prompt_inputs["prompt_token_ids"]), 8192) + + # [BI100] Adaptive max_tokens based on prompt length. + _prompt_len = len(prompt_inputs["prompt_token_ids"]) + if _prompt_len > 65536: + _adaptive_cap = 256 + elif _prompt_len > 32768: + _adaptive_cap = 512 + elif _prompt_len > 16384: + _adaptive_cap = 1024 + else: + _adaptive_cap = 8192 + + if request.max_tokens is not None: + request.max_tokens = min(request.max_tokens, _adaptive_cap) + + default_max_tokens = min(self.max_model_len - _prompt_len, + _adaptive_cap) if request.use_beam_search: sampling_params = request.to_beam_search_params( default_max_tokens) @@ -412,7 +427,7 @@ class OpenAIServingChat(OpenAIServing): prompt_adapter_request=prompt_adapter_request) engine_inputs = TokensPrompt( - prompt_token_ids=prompt_inputs["prompt_token_ids"]), 8192) + prompt_token_ids=prompt_inputs["prompt_token_ids"]) if mm_data is not None: engine_inputs["multi_modal_data"] = mm_data