[PD] Fix edge case and simplify large page size + chunked prefill (#5589)

This commit is contained in:
Byron Hsu
2025-04-21 10:27:02 -07:00
committed by GitHub
parent f29a718f63
commit deded17f38
3 changed files with 24 additions and 21 deletions

View File

@@ -1,4 +1,4 @@
prompt = "Hello " * 16000
prompt = [0] * 431
import json
@@ -6,8 +6,8 @@ import requests
response = requests.post(
"http://0.0.0.0:8000/generate",
json={"text": prompt, "sampling_params": {"temperature": 0}},
json={"input_ids": [prompt] * 32, "sampling_params": {"temperature": 0}},
)
print("Response content (raw):", response.content)
# print("Response content (raw):", response.content)