[Minor] Add some utility functions (#1671)
This commit is contained in:
@@ -710,7 +710,7 @@ class Scheduler:
|
||||
next_token_ids
|
||||
)
|
||||
|
||||
if logits_output:
|
||||
if batch.return_logprob:
|
||||
# Move logprobs to cpu
|
||||
if logits_output.next_token_logprobs is not None:
|
||||
logits_output.next_token_logprobs = (
|
||||
@@ -786,7 +786,7 @@ class Scheduler:
|
||||
self.num_generated_tokens += len(batch.reqs)
|
||||
|
||||
# Move logprobs to cpu
|
||||
if logits_output.next_token_logprobs is not None:
|
||||
if batch.return_logprob:
|
||||
next_token_logprobs = logits_output.next_token_logprobs[
|
||||
torch.arange(len(next_token_ids), device=next_token_ids.device),
|
||||
next_token_ids,
|
||||
|
||||
Reference in New Issue
Block a user