minor: some potential bugs (#1044)

This commit is contained in:
Liangsheng Yin
2024-08-11 22:35:44 -07:00
committed by GitHub
parent 14b6493087
commit fb7421db0d
2 changed files with 7 additions and 6 deletions

View File

@@ -2,7 +2,7 @@ from __future__ import annotations
"""Cache for chunked prefill, used when RadixCache is disabled."""
from typing import TYPE_CHECKING, Callable
from typing import TYPE_CHECKING, Callable, List, Optional
from sglang.srt.mem_cache.base_prefix_cache import BasePrefixCache
from sglang.srt.mem_cache.memory_pool import BaseTokenToKVPool, ReqToTokenPool
@@ -30,12 +30,13 @@ class ChunkCache(BasePrefixCache):
def reset(self):
self.entries = {}
def match_prefix(self, rid, **kwargs):
def match_prefix(self, rid: int, key: List[int]):
if rid not in self.entries:
return [], None
entry = self.entries[rid]
return entry.value, entry
max_prefix_len = len(key)
return entry.value[:max_prefix_len], entry
def cache_finished_req(self, req: Req, token_ids: Optional[List[int]] = None):
if token_ids is None: