EngineX-Hygon/sglang
5
0
Fork 0
Code Issues Pull Requests Actions 7 Projects Releases Wiki Activity

Enable overlap by default (#2067)

Browse Source
This commit is contained in:
Lianmin Zheng
2024-11-19 22:07:58 -08:00
committed by GitHub
parent 699384cb01
commit 7d671e4ad2
17 changed files with 92 additions and 75 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
python/sglang/srt/sampling/sampling_batch_info.py
View File

@@ -132,9 +132,6 @@ class SamplingBatchInfo:
return len(self.temperatures)
def update_penalties(self):
if not self.penalizer_orchestrator:
return
self.scaling_penalties = None
self.linear_penalties = None
@@ -176,8 +173,7 @@ class SamplingBatchInfo:
grammar.fill_vocab_mask(self.vocab_mask, i)
def filter_batch(self, unfinished_indices: List[int], new_indices: torch.Tensor):
if self.penalizer_orchestrator:
self.penalizer_orchestrator.filter(unfinished_indices, new_indices)
self.penalizer_orchestrator.filter(unfinished_indices, new_indices)
for item in [
"temperatures",
@@ -216,8 +212,7 @@ class SamplingBatchInfo:
return None
def merge_batch(self, other: "SamplingBatchInfo"):
if self.penalizer_orchestrator:
self.penalizer_orchestrator.merge(other.penalizer_orchestrator)
self.penalizer_orchestrator.merge(other.penalizer_orchestrator)
for item in [
"temperatures",