EngineX-Hygon/sglang
5
0
Fork 0
Code Issues Pull Requests Actions 7 Projects Releases Wiki Activity

Organize Attention Backends (#1547)

Browse Source
This commit is contained in:
Liangsheng Yin
2024-09-30 15:54:18 -07:00
committed by GitHub
parent 0f4fb19bc8
commit 99ec439da4
12 changed files with 229 additions and 205 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
python/sglang/srt/model_executor/forward_batch_info.py
View File

@@ -37,7 +37,7 @@ import numpy as np
import torch
if TYPE_CHECKING:
from sglang.srt.layers.attention_backend import AttentionBackend
from sglang.srt.layers.attention import AttentionBackend
from sglang.srt.managers.schedule_batch import ImageInputs, ModelWorkerBatch
from sglang.srt.mem_cache.memory_pool import BaseTokenToKVPool, ReqToTokenPool
from sglang.srt.model_executor.model_runner import ModelRunner

3
python/sglang/srt/model_executor/model_runner.py
View File

@@ -39,7 +39,8 @@ from vllm.model_executor.models import ModelRegistry
from sglang.srt.configs.model_config import AttentionArch, ModelConfig
from sglang.srt.constrained import disable_cache
from sglang.srt.layers.attention_backend import FlashInferAttnBackend, TritonAttnBackend
from sglang.srt.layers.attention.flashinfer_backend import FlashInferAttnBackend
from sglang.srt.layers.attention.triton_backend import TritonAttnBackend
from sglang.srt.layers.logits_processor import LogitsProcessorOutput
from sglang.srt.layers.sampler import Sampler
from sglang.srt.lora.lora_manager import LoRAManager