EngineX-Hygon/sglang
5
0
Fork 0
Code Issues Pull Requests Actions 7 Projects Releases Wiki Activity

Fix eagle on AMD (#7051)

Browse Source
This commit is contained in:
Lianmin Zheng
2025-06-10 05:22:40 -07:00
committed by GitHub
parent 2dae104dca
commit 019851d099
2 changed files with 4 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
python/sglang/srt/speculative/eagle_utils.py
View File

@@ -123,6 +123,9 @@ class EagleDraftInput:
cum_kv_seq_len = torch.zeros((bs + 1,), dtype=torch.int32, device="cuda")
cum_kv_seq_len[1:] = torch.cumsum(paged_kernel_lens, dim=0)
if paged_kernel_lens_sum is None:
paged_kernel_lens_sum = cum_kv_seq_len[-1]
kv_indices = torch.empty(
paged_kernel_lens_sum, dtype=torch.int32, device="cuda"
)

2
test/srt/test_bench_serving.py
View File

@@ -194,7 +194,7 @@ class TestBenchServing(CustomTestCase):
self.assertLess(res["median_ttft_ms"], 150)
# TODO: not set yet, need AMD machine
else:
self.assertLess(res["median_ttft_ms"], 94)
self.assertLess(res["median_ttft_ms"], 98)
self.assertLess(res["median_itl_ms"], 8)
def test_online_latency_eagle(self):