EngineX-Hygon/sglang
5
0
Fork 0
Code Issues Pull Requests Actions 7 Projects Releases Wiki Activity

Fix input_ids && rename to fill_ids (#1021)

Browse Source
This commit is contained in:
Liangsheng Yin
2024-08-10 16:24:12 -07:00
committed by GitHub
parent 54fb1c80c0
commit 43fbb6d919
7 changed files with 28 additions and 27 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
python/sglang/srt/mem_cache/radix_cache.py
View File

@@ -92,7 +92,7 @@ class RadixCache(BasePrefixCache):
def cache_finished_req(self, req: "Req", token_ids=None):
"""Cache request when it finishes."""
if token_ids is None:
token_ids = (req.input_ids + req.output_ids)[:-1]
token_ids = (req.origin_input_ids + req.output_ids)[:-1]
kv_indices = self.req_to_token_pool.req_to_token[
req.req_pool_idx, : len(token_ids)
]
@@ -116,7 +116,7 @@ class RadixCache(BasePrefixCache):
return
if token_ids is None:
token_ids = req.input_ids
token_ids = req.fill_ids
kv_indices = self.req_to_token_pool.req_to_token[
req.req_pool_idx, : len(token_ids)