[Minor] Add some utility functions (#1671)

This commit is contained in:
Lianmin Zheng
2024-10-14 20:08:03 -07:00
committed by GitHub
parent cd0be7489f
commit 4a292f670d
4 changed files with 42 additions and 2 deletions

View File

@@ -710,7 +710,7 @@ class Scheduler:
next_token_ids
)
if logits_output:
if batch.return_logprob:
# Move logprobs to cpu
if logits_output.next_token_logprobs is not None:
logits_output.next_token_logprobs = (
@@ -786,7 +786,7 @@ class Scheduler:
self.num_generated_tokens += len(batch.reqs)
# Move logprobs to cpu
if logits_output.next_token_logprobs is not None:
if batch.return_logprob:
next_token_logprobs = logits_output.next_token_logprobs[
torch.arange(len(next_token_ids), device=next_token_ids.device),
next_token_ids,