EngineX-Hygon/sglang
5
0
Fork 0
Code Issues Pull Requests Actions 7 Projects Releases Wiki Activity

Fix llava parallelism/fork bug (#315)

Browse Source
This commit is contained in:
Junlong Li
2024-03-29 10:24:54 +08:00
committed by GitHub
parent eddaa2b599
commit cb389c91bc
2 changed files with 4 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
python/sglang/backend/runtime_endpoint.py
View File

@@ -73,9 +73,11 @@ class RuntimeEndpoint(BaseBackend):
assert res.status_code == 200 assert res.status_code == 200
def commit_lazy_operations(self, s: StreamExecutor): def commit_lazy_operations(self, s: StreamExecutor):
data = {"text": s.text_, "sampling_params": {"max_new_tokens": 0}}
self._add_images(s, data)
res = http_request( res = http_request(
self.base_url + "/generate", self.base_url + "/generate",
json={"text": s.text_, "sampling_params": {"max_new_tokens": 0}}, json=data,
auth_token=self.auth_token, auth_token=self.auth_token,
api_key=self.api_key, api_key=self.api_key,
verify=self.verify, verify=self.verify,

1
python/sglang/lang/interpreter.py
View File

@@ -276,6 +276,7 @@ class StreamExecutor:
exes[i].messages_ = list(self.messages_) exes[i].messages_ = list(self.messages_)
exes[i].cur_role = self.cur_role exes[i].cur_role = self.cur_role
exes[i].fork_start_text_pos = len(self.text_) exes[i].fork_start_text_pos = len(self.text_)
exes[i].images_ = list(self.images_)
return exes return exes