168ad600b5
### What this PR does / why we need it?
For offline scenarios, adjust the scheduling process to prioritize the
prefill phase of all requests, then process the decode phase of all
requests.
### How was this patch tested?
```
max_num_seqs=24,
additional_config={
"ascend_scheduler_config":{
"enabled": True,
"enable_pd_transfer": True,
"decode_max_num_seqs": 24,
"enable_chunked_prefill": False
}
},
```
| input | output | num prompts | max_num_seqs | dp | tp | scheduler |
tps |
| ------ | ------ | ---------- | ---------------- | ---- | ---- |
---------------- | --------------- |
| dapo-math-17K | 2K | 384 | 24 | 2 | 1 | v1 | 234.06 |
| dapo-math-17K | 2K | 384 | 24 | 2 | 1 | pd transfer | 239.59(+2.4%) |
| dapo-math-17K| 2K | 384 | 24 | 4 | 1 | v1 | 222.85 |
| dapo-math-17K| 2K | 384 | 24 | 4 | 1 | pd transfer | 225.81(+1.3%) |
- vLLM version: v0.10.1.1
- vLLM main:
6fb2788163
---------
Signed-off-by: CaranLic <740821011@qq.com>
51 lines
1.8 KiB
Python
51 lines
1.8 KiB
Python
import itertools
|
|
from collections.abc import Sequence
|
|
from typing import TYPE_CHECKING, Union
|
|
|
|
import torch
|
|
from vllm.logger import init_logger
|
|
from vllm.v1.sample import logits_processor
|
|
from vllm.v1.sample.logits_processor.builtin import (LogitBiasLogitsProcessor,
|
|
MinTokensLogitsProcessor)
|
|
from vllm.v1.sample.logits_processor.interface import LogitsProcessor
|
|
from vllm.v1.sample.logits_processor.state import LogitsProcessors
|
|
|
|
from vllm_ascend.sample.logits_processor.builtin import \
|
|
AscendMinPLogitsProcessor
|
|
|
|
if TYPE_CHECKING:
|
|
from vllm.config import VllmConfig
|
|
|
|
logger = init_logger(__name__)
|
|
|
|
# Error message when the user tries to initialize vLLM with a pooling model
|
|
# and custom logitsproces
|
|
STR_POOLING_REJECTS_LOGITSPROCS = ("Pooling models do not support custom"
|
|
" logits processors.")
|
|
|
|
BUILTIN_LOGITS_PROCESSORS: list[type[LogitsProcessor]] = [
|
|
MinTokensLogitsProcessor,
|
|
LogitBiasLogitsProcessor,
|
|
AscendMinPLogitsProcessor,
|
|
]
|
|
|
|
|
|
def build_logitsprocs(
|
|
vllm_config: "VllmConfig",
|
|
device: torch.device,
|
|
is_pin_memory: bool,
|
|
is_pooling_model: bool,
|
|
custom_logitsprocs: Sequence[Union[str, type[LogitsProcessor]]] = (),
|
|
) -> LogitsProcessors:
|
|
if is_pooling_model:
|
|
if custom_logitsprocs:
|
|
raise ValueError(STR_POOLING_REJECTS_LOGITSPROCS)
|
|
logger.debug("Skipping logits processor loading because pooling models"
|
|
" do not support logits processors.")
|
|
return LogitsProcessors()
|
|
custom_logitsprocs_classes = logits_processor._load_custom_logitsprocs(
|
|
custom_logitsprocs)
|
|
return LogitsProcessors(
|
|
ctor(vllm_config, device, is_pin_memory) for ctor in itertools.chain(
|
|
BUILTIN_LOGITS_PROCESSORS, custom_logitsprocs_classes))
|