EngineX-Hygon/sglang
5
0
Fork 0
Code Issues Pull Requests Actions 7 Projects Releases Wiki Activity

Revert "Temporary fix invalid sample results" (#673)

Browse Source
This commit is contained in:
Liangsheng Yin
2024-07-20 02:06:31 -07:00
committed by GitHub
parent 9592a1f3bd
commit 39c57317e1
1 changed files with 0 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
python/sglang/srt/managers/controller/infer_batch.py
View File

@@ -673,16 +673,6 @@ class Batch:
batch_next_token_ids, _ = top_k_top_p_sampling_from_probs( batch_next_token_ids, _ = top_k_top_p_sampling_from_probs(
probs, uniform_samples, self.top_ks, self.top_ps probs, uniform_samples, self.top_ks, self.top_ps
) )
# FIXME: This is a temporary fix for the illegal token ids in sampling.
illegal_mask = (
batch_next_token_ids < 0 or batch_next_token_ids >= probs.shape[-1]
)
if torch.any(illegal_mask):
warnings.warn("Illegal token ids in sampling.")
batch_next_token_ids = torch.where(
illegal_mask, torch.argmax(probs, dim=-1), batch_next_token_ids
)
except RuntimeError as e: except RuntimeError as e:
warnings.warn(f"Ignore errors in sampling: {e}") warnings.warn(f"Ignore errors in sampling: {e}")
batch_next_token_ids = torch.argmax(probs, dim=-1) batch_next_token_ids = torch.argmax(probs, dim=-1)