Minor: style improvement of radix_cache and memory_pool (#395)

2024-04-26 01:01:36 +08:00
parent 9216b10678
commit 19818b9c2f
6 changed files with 26 additions and 37 deletions
--- a/test/srt/model/test_llama_low_api.py
+++ b/test/srt/model/test_llama_low_api.py
@@ -105,7 +105,7 @@ def test_generate_worker(

    for i in range(batch_size):
        req_idx = req_pool_indices[i].item()
-        model.token_to_kv_pool.free(
+        model.token_to_kv_pool.dec_refs(
            model.req_to_token_pool.req_to_token[req_idx, : seq_lens[i]]
        )
    model.req_to_token_pool.free(req_pool_indices)