feat: add check_env (#645)

This commit is contained in:
zhyncs
2024-07-18 14:39:28 +10:00
committed by GitHub
parent 476584cb6e
commit d93388da3e
5 changed files with 181 additions and 6 deletions

View File

@@ -44,7 +44,14 @@ class ReqToTokenPool:
class TokenToKVPool:
"""A memory pool that maps a token to its kv cache locations"""
def __init__(self, size: int, dtype: torch.dtype, head_num: int, head_dim: int, layer_num: int):
def __init__(
self,
size: int,
dtype: torch.dtype,
head_num: int,
head_dim: int,
layer_num: int,
):
self.size = size
# We also add one slot. This slot is used for writing dummy output from padded tokens.