Remove the unused write_with_records (#2972)
This commit is contained in:
@@ -158,7 +158,6 @@ class ImageInputs:
|
|||||||
im_end_id: Optional[torch.Tensor] = None
|
im_end_id: Optional[torch.Tensor] = None
|
||||||
slice_start_id: Optional[torch.Tensor] = None
|
slice_start_id: Optional[torch.Tensor] = None
|
||||||
slice_end_id: Optional[torch.Tensor] = None
|
slice_end_id: Optional[torch.Tensor] = None
|
||||||
|
|
||||||
tgt_sizes: Optional[list] = None
|
tgt_sizes: Optional[list] = None
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
|
|||||||
@@ -49,7 +49,6 @@ class ReqToTokenPool:
|
|||||||
size: int,
|
size: int,
|
||||||
max_context_len: int,
|
max_context_len: int,
|
||||||
device: str,
|
device: str,
|
||||||
use_records: bool,
|
|
||||||
enable_memory_saver: bool,
|
enable_memory_saver: bool,
|
||||||
):
|
):
|
||||||
memory_saver_adapter = TorchMemorySaverAdapter.create(
|
memory_saver_adapter = TorchMemorySaverAdapter.create(
|
||||||
@@ -64,17 +63,9 @@ class ReqToTokenPool:
|
|||||||
(size, max_context_len), dtype=torch.int32, device=device
|
(size, max_context_len), dtype=torch.int32, device=device
|
||||||
)
|
)
|
||||||
self.free_slots = list(range(size))
|
self.free_slots = list(range(size))
|
||||||
self.write_records = []
|
|
||||||
self.use_records = use_records
|
|
||||||
|
|
||||||
if self.use_records:
|
|
||||||
self.write = self.write_with_records
|
|
||||||
else:
|
|
||||||
self.write = self.write_without_records
|
|
||||||
|
|
||||||
def write(self, indices, values):
|
def write(self, indices, values):
|
||||||
# Keep the signature for type checking. It will be assigned during runtime.
|
self.req_to_token[indices] = values
|
||||||
raise NotImplementedError()
|
|
||||||
|
|
||||||
def available_size(self):
|
def available_size(self):
|
||||||
return len(self.free_slots)
|
return len(self.free_slots)
|
||||||
@@ -96,23 +87,6 @@ class ReqToTokenPool:
|
|||||||
|
|
||||||
def clear(self):
|
def clear(self):
|
||||||
self.free_slots = list(range(self.size))
|
self.free_slots = list(range(self.size))
|
||||||
self.write_records = []
|
|
||||||
|
|
||||||
def write_without_records(self, indices, values):
|
|
||||||
self.req_to_token[indices] = values
|
|
||||||
|
|
||||||
def write_with_records(self, indices, values):
|
|
||||||
self.req_to_token[indices] = values
|
|
||||||
self.write_records.append((indices, values))
|
|
||||||
|
|
||||||
def get_write_records(self):
|
|
||||||
ret = self.write_records
|
|
||||||
self.write_records = []
|
|
||||||
return ret
|
|
||||||
|
|
||||||
def apply_write_records(self, write_records: List[Tuple]):
|
|
||||||
for indices, values in write_records:
|
|
||||||
self.req_to_token[indices] = values
|
|
||||||
|
|
||||||
|
|
||||||
class BaseTokenToKVPool:
|
class BaseTokenToKVPool:
|
||||||
|
|||||||
@@ -617,7 +617,6 @@ class ModelRunner:
|
|||||||
size=max_num_reqs + 1,
|
size=max_num_reqs + 1,
|
||||||
max_context_len=self.model_config.context_len + 4,
|
max_context_len=self.model_config.context_len + 4,
|
||||||
device=self.device,
|
device=self.device,
|
||||||
use_records=False,
|
|
||||||
enable_memory_saver=self.server_args.enable_memory_saver,
|
enable_memory_saver=self.server_args.enable_memory_saver,
|
||||||
)
|
)
|
||||||
if (
|
if (
|
||||||
|
|||||||
Reference in New Issue
Block a user