EngineX-Hygon/sglang
5
0
Fork 0
Code Issues Pull Requests Actions 7 Projects Releases Wiki Activity

Add SSL Cert Functionality (#224)

Browse Source
This commit is contained in:
Srinivas Billa
2024-03-03 09:41:41 +00:00
committed by GitHub
parent dfb13ac455
commit 01b07ea3ac
2 changed files with 15 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files

17
python/sglang/backend/runtime_endpoint.py
View File

@@ -12,15 +12,16 @@ from sglang.utils import encode_image_base64, find_printable_text, http_request
class RuntimeEndpoint(BaseBackend): class RuntimeEndpoint(BaseBackend):
def __init__(self, base_url, auth_token=None): def __init__(self, base_url, auth_token=None, verify=None):
super().__init__() super().__init__()
self.support_concate_and_append = True self.support_concate_and_append = True
self.base_url = base_url self.base_url = base_url
self.auth_token = auth_token self.auth_token = auth_token
self.verify = verify
res = http_request( res = http_request(
self.base_url + "/get_model_info", auth_token=self.auth_token self.base_url + "/get_model_info", auth_token=self.auth_token, verify=self.verify
) )
assert res.status_code == 200 assert res.status_code == 200
self.model_info = res.json() self.model_info = res.json()
@@ -40,6 +41,7 @@ class RuntimeEndpoint(BaseBackend):
self.base_url + "/generate", self.base_url + "/generate",
json={"text": prefix_str, "sampling_params": {"max_new_tokens": 0}}, json={"text": prefix_str, "sampling_params": {"max_new_tokens": 0}},
auth_token=self.auth_token, auth_token=self.auth_token,
verify=self.verify
) )
assert res.status_code == 200 assert res.status_code == 200
@@ -48,6 +50,7 @@ class RuntimeEndpoint(BaseBackend):
self.base_url + "/generate", self.base_url + "/generate",
json={"text": s.text_, "sampling_params": {"max_new_tokens": 0}}, json={"text": s.text_, "sampling_params": {"max_new_tokens": 0}},
auth_token=self.auth_token, auth_token=self.auth_token,
verify=self.verify
) )
assert res.status_code == 200 assert res.status_code == 200
@@ -55,7 +58,7 @@ class RuntimeEndpoint(BaseBackend):
data = {"text": s.text_, "sampling_params": {"max_new_tokens": 0}} data = {"text": s.text_, "sampling_params": {"max_new_tokens": 0}}
self._add_images(s, data) self._add_images(s, data)
res = http_request( res = http_request(
self.base_url + "/generate", json=data, auth_token=self.auth_token self.base_url + "/generate", json=data, auth_token=self.auth_token, verify=self.verify
) )
assert res.status_code == 200 assert res.status_code == 200
@@ -87,7 +90,7 @@ class RuntimeEndpoint(BaseBackend):
self._add_images(s, data) self._add_images(s, data)
res = http_request( res = http_request(
self.base_url + "/generate", json=data, auth_token=self.auth_token self.base_url + "/generate", json=data, auth_token=self.auth_token, verify=self.verify
) )
obj = res.json() obj = res.json()
comp = obj["text"] comp = obj["text"]
@@ -126,6 +129,7 @@ class RuntimeEndpoint(BaseBackend):
json=data, json=data,
stream=True, stream=True,
auth_token=self.auth_token, auth_token=self.auth_token,
verify=self.verify
) )
pos = 0 pos = 0
@@ -157,7 +161,7 @@ class RuntimeEndpoint(BaseBackend):
data = {"text": s.text_, "sampling_params": {"max_new_tokens": 0}} data = {"text": s.text_, "sampling_params": {"max_new_tokens": 0}}
self._add_images(s, data) self._add_images(s, data)
res = http_request( res = http_request(
self.base_url + "/generate", json=data, auth_token=self.auth_token self.base_url + "/generate", json=data, auth_token=self.auth_token, verify=self.verify
) )
assert res.status_code == 200 assert res.status_code == 200
prompt_len = res.json()["meta_info"]["prompt_tokens"] prompt_len = res.json()["meta_info"]["prompt_tokens"]
@@ -171,7 +175,7 @@ class RuntimeEndpoint(BaseBackend):
} }
self._add_images(s, data) self._add_images(s, data)
res = http_request( res = http_request(
self.base_url + "/generate", json=data, auth_token=self.auth_token self.base_url + "/generate", json=data, auth_token=self.auth_token, verify=self.verify
) )
assert res.status_code == 200 assert res.status_code == 200
obj = res.json() obj = res.json()
@@ -188,6 +192,7 @@ class RuntimeEndpoint(BaseBackend):
self.base_url + "/concate_and_append_request", self.base_url + "/concate_and_append_request",
json={"src_rids": src_rids, "dst_rid": dst_rid}, json={"src_rids": src_rids, "dst_rid": dst_rid},
auth_token=self.auth_token, auth_token=self.auth_token,
verify=self.verify
) )
assert res.status_code == 200 assert res.status_code == 200

8
python/sglang/utils.py
View File

@@ -88,16 +88,16 @@ class HttpResponse:
return self.resp.status return self.resp.status
def http_request(url, json=None, stream=False, auth_token=None): def http_request(url, json=None, stream=False, auth_token=None, verify=None):
"""A faster version of requests.post with low-level urllib API.""" """A faster version of requests.post with low-level urllib API."""
if stream: if stream:
if auth_token is None: if auth_token is None:
return requests.post(url, json=json, stream=True) return requests.post(url, json=json, stream=True, verify=verify)
headers = { headers = {
"Content-Type": "application/json", "Content-Type": "application/json",
"Authentication": f"Bearer {auth_token}", "Authentication": f"Bearer {auth_token}",
} }
return requests.post(url, json=json, stream=True, headers=headers) return requests.post(url, json=json, stream=True, headers=headers, verify=verify)
else: else:
req = urllib.request.Request(url) req = urllib.request.Request(url)
req.add_header("Content-Type", "application/json; charset=utf-8") req.add_header("Content-Type", "application/json; charset=utf-8")
@@ -107,7 +107,7 @@ def http_request(url, json=None, stream=False, auth_token=None):
data = None data = None
else: else:
data = bytes(dumps(json), encoding="utf-8") data = bytes(dumps(json), encoding="utf-8")
resp = urllib.request.urlopen(req, data=data) resp = urllib.request.urlopen(req, data=data, cafile=verify)
return HttpResponse(resp) return HttpResponse(resp)