EngineX-Hygon/sglang
5
0
Fork 0
Code Issues Pull Requests Actions 7 Projects Releases Wiki Activity

Test openai vision api (#925)

Browse Source
This commit is contained in:
Ying Sheng
2024-08-04 20:51:55 -07:00
committed by GitHub
parent ebf69964cd
commit 3bc99e6fe4
13 changed files with 102 additions and 228 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
test/lang/test_bind_cache.py
View File

@@ -1,7 +1,7 @@
import unittest
import sglang as sgl
from sglang.test.test_utils import MODEL_NAME_FOR_TEST
from sglang.test.test_utils import DEFAULT_MODEL_NAME_FOR_TEST
class TestBind(unittest.TestCase):
@@ -9,7 +9,7 @@ class TestBind(unittest.TestCase):
@classmethod
def setUpClass(cls):
cls.backend = sgl.Runtime(model_path=MODEL_NAME_FOR_TEST)
cls.backend = sgl.Runtime(model_path=DEFAULT_MODEL_NAME_FOR_TEST)
sgl.set_default_backend(cls.backend)
@classmethod

4
test/lang/test_srt_backend.py
View File

@@ -14,7 +14,7 @@ from sglang.test.test_programs import (
test_stream,
test_tool_use,
)
from sglang.test.test_utils import MODEL_NAME_FOR_TEST
from sglang.test.test_utils import DEFAULT_MODEL_NAME_FOR_TEST
class TestSRTBackend(unittest.TestCase):
@@ -22,7 +22,7 @@ class TestSRTBackend(unittest.TestCase):
@classmethod
def setUpClass(cls):
cls.backend = sgl.Runtime(model_path=MODEL_NAME_FOR_TEST)
cls.backend = sgl.Runtime(model_path=DEFAULT_MODEL_NAME_FOR_TEST)
sgl.set_default_backend(cls.backend)
@classmethod

3
test/srt/run_suite.py
View File

@@ -5,8 +5,9 @@ from sglang.test.test_utils import run_unittest_files
suites = {
"minimal": [
"test_openai_server.py",
"test_eval_accuracy.py",
"test_openai_server.py",
"test_vision_openai_server.py",
"test_chunked_prefill.py",
"test_torch_compile.py",
"models/test_causal_models.py",

4
test/srt/test_chunked_prefill.py
View File

@@ -3,14 +3,14 @@ from types import SimpleNamespace
from sglang.srt.utils import kill_child_process
from sglang.test.run_eval import run_eval
from sglang.test.test_utils import MODEL_NAME_FOR_TEST, popen_launch_server
from sglang.test.test_utils import DEFAULT_MODEL_NAME_FOR_TEST, popen_launch_server
class TestAccuracy(unittest.TestCase):
@classmethod
def setUpClass(cls):
cls.model = MODEL_NAME_FOR_TEST
cls.model = DEFAULT_MODEL_NAME_FOR_TEST
cls.base_url = f"http://localhost:8157"
cls.process = popen_launch_server(
cls.model,

4
test/srt/test_eval_accuracy.py
View File

@@ -3,14 +3,14 @@ from types import SimpleNamespace
from sglang.srt.utils import kill_child_process
from sglang.test.run_eval import run_eval
from sglang.test.test_utils import MODEL_NAME_FOR_TEST, popen_launch_server
from sglang.test.test_utils import DEFAULT_MODEL_NAME_FOR_TEST, popen_launch_server
class TestAccuracy(unittest.TestCase):
@classmethod
def setUpClass(cls):
cls.model = MODEL_NAME_FOR_TEST
cls.model = DEFAULT_MODEL_NAME_FOR_TEST
cls.base_url = f"http://localhost:8157"
cls.process = popen_launch_server(cls.model, cls.base_url, timeout=300)

8
test/srt/test_openai_server.py
View File

@@ -5,21 +5,21 @@ import openai
from sglang.srt.hf_transformers_utils import get_tokenizer
from sglang.srt.utils import kill_child_process
from sglang.test.test_utils import MODEL_NAME_FOR_TEST, popen_launch_server
from sglang.test.test_utils import DEFAULT_MODEL_NAME_FOR_TEST, popen_launch_server
class TestOpenAIServer(unittest.TestCase):
@classmethod
def setUpClass(cls):
cls.model = MODEL_NAME_FOR_TEST
cls.model = DEFAULT_MODEL_NAME_FOR_TEST
cls.base_url = f"http://localhost:8157"
cls.api_key = "sk-123456"
cls.process = popen_launch_server(
cls.model, cls.base_url, timeout=300, api_key=cls.api_key
)
cls.base_url += "/v1"
cls.tokenizer = get_tokenizer(MODEL_NAME_FOR_TEST)
cls.tokenizer = get_tokenizer(DEFAULT_MODEL_NAME_FOR_TEST)
@classmethod
def tearDownClass(cls):
@@ -147,6 +147,7 @@ class TestOpenAIServer(unittest.TestCase):
top_logprobs=logprobs,
n=parallel_sample_num,
)
if logprobs:
assert isinstance(
response.choices[0].logprobs.content[0].top_logprobs[0].token, str
@@ -158,6 +159,7 @@ class TestOpenAIServer(unittest.TestCase):
assert (
ret_num_top_logprobs == logprobs
), f"{ret_num_top_logprobs} vs {logprobs}"
assert len(response.choices) == parallel_sample_num
assert response.choices[0].message.role == "assistant"
assert isinstance(response.choices[0].message.content, str)

4
test/srt/test_srt_endpoint.py
View File

@@ -5,14 +5,14 @@ import requests
from sglang.srt.utils import kill_child_process
from sglang.test.run_eval import run_eval
from sglang.test.test_utils import MODEL_NAME_FOR_TEST, popen_launch_server
from sglang.test.test_utils import DEFAULT_MODEL_NAME_FOR_TEST, popen_launch_server
class TestSRTEndpoint(unittest.TestCase):
@classmethod
def setUpClass(cls):
cls.model = MODEL_NAME_FOR_TEST
cls.model = DEFAULT_MODEL_NAME_FOR_TEST
cls.base_url = f"http://localhost:{8157}"
cls.process = popen_launch_server(cls.model, cls.base_url, timeout=300)

4
test/srt/test_torch_compile.py
View File

@@ -3,14 +3,14 @@ from types import SimpleNamespace
from sglang.srt.utils import kill_child_process
from sglang.test.run_eval import run_eval
from sglang.test.test_utils import MODEL_NAME_FOR_TEST, popen_launch_server
from sglang.test.test_utils import DEFAULT_MODEL_NAME_FOR_TEST, popen_launch_server
class TestAccuracy(unittest.TestCase):
@classmethod
def setUpClass(cls):
cls.model = MODEL_NAME_FOR_TEST
cls.model = DEFAULT_MODEL_NAME_FOR_TEST
cls.base_url = f"http://localhost:8157"
cls.process = popen_launch_server(
cls.model, cls.base_url, timeout=300, other_args=["--enable-torch-compile"]

75
test/srt/test_vision_openai_server.py Normal file
View File

@@ -0,0 +1,75 @@
import json
import unittest
import openai
from sglang.srt.hf_transformers_utils import get_tokenizer
from sglang.srt.utils import kill_child_process
from sglang.test.test_utils import popen_launch_server
class TestOpenAIVisionServer(unittest.TestCase):
@classmethod
def setUpClass(cls):
cls.model = "liuhaotian/llava-v1.6-vicuna-7b"
cls.base_url = "http://localhost:8157"
cls.api_key = "sk-123456"
cls.process = popen_launch_server(
cls.model,
cls.base_url,
timeout=300,
api_key=cls.api_key,
other_args=[
"--chat-template",
"vicuna_v1.1",
"--tokenizer-path",
"llava-hf/llava-1.5-7b-hf",
"--log-requests",
],
)
cls.base_url += "/v1"
@classmethod
def tearDownClass(cls):
kill_child_process(cls.process.pid)
def test_chat_completion(self):
client = openai.Client(api_key=self.api_key, base_url=self.base_url)
response = client.chat.completions.create(
model="default",
messages=[
{
"role": "user",
"content": [
{
"type": "image_url",
"image_url": {
"url": "https://github.com/sgl-project/sglang/blob/main/assets/logo.png?raw=true"
},
},
{"type": "text", "text": "Describe this image"},
],
},
],
temperature=0,
max_tokens=32,
)
assert response.choices[0].message.role == "assistant"
assert isinstance(response.choices[0].message.content, str)
assert response.id
assert response.created
assert response.usage.prompt_tokens > 0
assert response.usage.completion_tokens > 0
assert response.usage.total_tokens > 0
if __name__ == "__main__":
unittest.main(warnings="ignore")
# t = TestOpenAIVisionServer()
# t.setUpClass()
# t.test_chat_completion()
# t.tearDownClass()