Misc fix for min_p_sampling, --cuda-graph-bs (#2761)

2025-01-07 02:52:53 -08:00
parent 6d08ce2aa9
commit bdc1acf6cd
17 changed files with 135 additions and 63 deletions
--- a/python/sglang/srt/sampling/sampling_batch_info.py
+++ b/python/sglang/srt/sampling/sampling_batch_info.py
@@ -232,6 +232,7 @@ class SamplingBatchInfo:
        self.logit_bias = SamplingBatchInfo.merge_bias_tensor(
            self.logit_bias, other.logit_bias, len(self), len(other), self.device
        )
+        self.need_min_p_sampling = self.need_min_p_sampling or other.need_min_p_sampling

    def apply_logits_bias(self, logits: torch.Tensor):
        # Apply logit_bias