@@ -0,0 +1,47 @@
|
||||
import pytest
|
||||
|
||||
from tests.e2e.conftest import RemoteOpenAIServer
|
||||
from tests.e2e.weekly.single_node.engine_func_test_robot.utility.http_client import (
|
||||
HTTPClient,
|
||||
)
|
||||
|
||||
env_dict: dict = {}
|
||||
|
||||
server_args: list = [
|
||||
"--served-model-name",
|
||||
"auto",
|
||||
"--max-model-len",
|
||||
"65536",
|
||||
"--tensor-parallel-size",
|
||||
"2",
|
||||
"--enable-expert-parallel",
|
||||
"--allowed-local-media-path",
|
||||
"/",
|
||||
"--limit-mm-per-prompt.video",
|
||||
"1",
|
||||
"--limit-mm-per-prompt.image",
|
||||
"5",
|
||||
"--enable-auto-tool-choice",
|
||||
"--tool-call-parser",
|
||||
"hermes",
|
||||
"--safetensors-load-strategy",
|
||||
"prefetch",
|
||||
]
|
||||
|
||||
|
||||
@pytest.fixture(scope="session")
|
||||
def api_client(request):
|
||||
model = "Qwen/Qwen3-VL-30B-A3B-Instruct"
|
||||
|
||||
with RemoteOpenAIServer(model, server_args, server_port=8000, env_dict=env_dict, auto_port=False) as server:
|
||||
yield HTTPClient(base_url=server.url_root)
|
||||
|
||||
|
||||
def pytest_addoption(parser):
|
||||
parser.addoption("--thinkTagOutput", action="store", type=str, default="false", required=False)
|
||||
parser.addoption("--engineArchitecture", action="store", default="single", choices=["pd", "single"])
|
||||
parser.addoption("--maxModelLength", action="store", default="128")
|
||||
parser.addoption("--model", action="store", default="qwen")
|
||||
parser.addoption("--imageNum", action="store", type=int, default=1)
|
||||
parser.addoption("--videoNum", action="store", type=int, default=1)
|
||||
parser.addoption("--audioNum", action="store", type=int, default=1)
|
||||
@@ -0,0 +1 @@
|
||||
# chat_template_kwargs field test package
|
||||
@@ -0,0 +1,184 @@
|
||||
from tests.e2e.weekly.single_node.engine_func_test_robot.utility import assertion
|
||||
from tests.e2e.weekly.single_node.engine_func_test_robot.utility import (
|
||||
request_helper as helper,
|
||||
)
|
||||
|
||||
|
||||
def test_chat_template_kwargs_string_non_stream(api_client, request):
|
||||
"""Non-streaming: chat_template_kwargs is a string instead of an object, so a 400 error should be returned."""
|
||||
request_body = {
|
||||
"model": "auto",
|
||||
"messages": [{"role": "user", "content": "你好"}],
|
||||
"chat_template_kwargs": "invalid_string",
|
||||
"stream": False,
|
||||
"max_tokens": 512,
|
||||
}
|
||||
|
||||
response = helper.send_request(api_client, "/v1/chat/completions", request_body)
|
||||
|
||||
# Check: status code is 400 and error code is 400
|
||||
assertion.assert_status_code_400(response)
|
||||
assertion.assert_error_code_400(response)
|
||||
|
||||
|
||||
def test_chat_template_kwargs_string_stream(api_client, request):
|
||||
"""Streaming: chat_template_kwargs is a string, so a 400 error should be returned."""
|
||||
request_body = {
|
||||
"model": "auto",
|
||||
"messages": [{"role": "user", "content": "你好"}],
|
||||
"chat_template_kwargs": "invalid_string",
|
||||
"stream": True,
|
||||
"max_tokens": 512,
|
||||
}
|
||||
|
||||
response = helper.send_request(api_client, "/v1/chat/completions", request_body)
|
||||
|
||||
# Check: status code is 400 and error code is 400
|
||||
assertion.assert_status_code_400(response)
|
||||
assertion.assert_error_code_400(response)
|
||||
|
||||
|
||||
def test_chat_template_kwargs_array_non_stream(api_client, request):
|
||||
"""Non-streaming: chat_template_kwargs is an array instead of an object, so a 400 error should be returned."""
|
||||
request_body = {
|
||||
"model": "auto",
|
||||
"messages": [{"role": "user", "content": "你好"}],
|
||||
"chat_template_kwargs": ["item1", "item2"],
|
||||
"stream": False,
|
||||
"max_tokens": 512,
|
||||
}
|
||||
|
||||
response = helper.send_request(api_client, "/v1/chat/completions", request_body)
|
||||
|
||||
# Check: status code is 400 and error code is 400
|
||||
assertion.assert_status_code_400(response)
|
||||
assertion.assert_error_code_400(response)
|
||||
|
||||
|
||||
def test_chat_template_kwargs_array_stream(api_client, request):
|
||||
"""Streaming: chat_template_kwargs is an array, so a 400 error should be returned."""
|
||||
request_body = {
|
||||
"model": "auto",
|
||||
"messages": [{"role": "user", "content": "你好"}],
|
||||
"chat_template_kwargs": ["item1", "item2"],
|
||||
"stream": True,
|
||||
"max_tokens": 512,
|
||||
}
|
||||
|
||||
response = helper.send_request(api_client, "/v1/chat/completions", request_body)
|
||||
|
||||
# Check: status code is 400 and error code is 400
|
||||
assertion.assert_status_code_400(response)
|
||||
assertion.assert_error_code_400(response)
|
||||
|
||||
|
||||
def test_chat_template_kwargs_integer_non_stream(api_client, request):
|
||||
"""Non-streaming: chat_template_kwargs is an integer, so a 400 error should be returned."""
|
||||
request_body = {
|
||||
"model": "auto",
|
||||
"messages": [{"role": "user", "content": "你好"}],
|
||||
"chat_template_kwargs": 123,
|
||||
"stream": False,
|
||||
"max_tokens": 512,
|
||||
}
|
||||
|
||||
response = helper.send_request(api_client, "/v1/chat/completions", request_body)
|
||||
|
||||
# Check: status code is 400 and error code is 400
|
||||
assertion.assert_status_code_400(response)
|
||||
assertion.assert_error_code_400(response)
|
||||
|
||||
|
||||
def test_chat_template_kwargs_integer_stream(api_client, request):
|
||||
"""Streaming: chat_template_kwargs is an integer, so a 400 error should be returned."""
|
||||
request_body = {
|
||||
"model": "auto",
|
||||
"messages": [{"role": "user", "content": "你好"}],
|
||||
"chat_template_kwargs": 123,
|
||||
"stream": True,
|
||||
"max_tokens": 512,
|
||||
}
|
||||
|
||||
response = helper.send_request(api_client, "/v1/chat/completions", request_body)
|
||||
|
||||
# Check: status code is 400 and error code is 400
|
||||
assertion.assert_status_code_400(response)
|
||||
assertion.assert_error_code_400(response)
|
||||
|
||||
|
||||
def test_chat_template_kwargs_boolean_non_stream(api_client, request):
|
||||
"""Non-streaming: chat_template_kwargs is a boolean, so a 400 error should be returned."""
|
||||
request_body = {
|
||||
"model": "auto",
|
||||
"messages": [{"role": "user", "content": "你好"}],
|
||||
"chat_template_kwargs": True,
|
||||
"stream": False,
|
||||
"max_tokens": 512,
|
||||
}
|
||||
|
||||
response = helper.send_request(api_client, "/v1/chat/completions", request_body)
|
||||
|
||||
# Check: status code is 400 and error code is 400
|
||||
assertion.assert_status_code_400(response)
|
||||
assertion.assert_error_code_400(response)
|
||||
|
||||
|
||||
def test_chat_template_kwargs_boolean_stream(api_client, request):
|
||||
"""Streaming: chat_template_kwargs is a boolean, so a 400 error should be returned."""
|
||||
request_body = {
|
||||
"model": "auto",
|
||||
"messages": [{"role": "user", "content": "你好"}],
|
||||
"chat_template_kwargs": False,
|
||||
"stream": True,
|
||||
"max_tokens": 512,
|
||||
}
|
||||
|
||||
response = helper.send_request(api_client, "/v1/chat/completions", request_body)
|
||||
|
||||
# Check: status code is 400 and error code is 400
|
||||
assertion.assert_status_code_400(response)
|
||||
assertion.assert_error_code_400(response)
|
||||
|
||||
|
||||
def test_chat_template_kwargs_nested_invalid_type_non_stream(api_client, request):
|
||||
"""Non-streaming: chat_template_kwargs contains a nested invalid value type, so a 400 error should be returned."""
|
||||
request_body = {
|
||||
"model": "auto",
|
||||
"messages": [{"role": "user", "content": "你好"}],
|
||||
"chat_template_kwargs": {
|
||||
"valid_param": "value",
|
||||
"invalid_param": [1, 2, 3], # Some engines may not support array values for parameters
|
||||
},
|
||||
"stream": False,
|
||||
"max_tokens": 512,
|
||||
}
|
||||
|
||||
response = helper.send_request(api_client, "/v1/chat/completions", request_body)
|
||||
|
||||
# Check: should be 400 if the engine validates strictly, or 200 if it ignores invalid values
|
||||
assert response.status_code in [
|
||||
200,
|
||||
400,
|
||||
], f"status code should be 200 or 400, got {response.status_code}"
|
||||
|
||||
|
||||
def test_chat_template_kwargs_nested_invalid_type_stream(api_client, request):
|
||||
"""Streaming: chat_template_kwargs contains a nested invalid value type."""
|
||||
request_body = {
|
||||
"model": "auto",
|
||||
"messages": [{"role": "user", "content": "你好"}],
|
||||
"chat_template_kwargs": {
|
||||
"valid_param": "value",
|
||||
"invalid_param": {"nested": [1, 2, 3]},
|
||||
},
|
||||
"stream": True,
|
||||
"max_tokens": 512,
|
||||
}
|
||||
|
||||
response = helper.send_request(api_client, "/v1/chat/completions", request_body)
|
||||
|
||||
# Check: status code should be 200 or 400
|
||||
assert response.status_code in [
|
||||
200,
|
||||
400,
|
||||
], f"status code should be 200 or 400, got {response.status_code}"
|
||||
@@ -0,0 +1,244 @@
|
||||
import pytest
|
||||
|
||||
from tests.e2e.weekly.single_node.engine_func_test_robot.utility import assertion
|
||||
from tests.e2e.weekly.single_node.engine_func_test_robot.utility import (
|
||||
request_helper as helper,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("stream", [False, True], ids=["non_stream", "stream"])
|
||||
def test_chat_template_kwargs_very_long_value(api_client, request, stream):
|
||||
"""chat_template_kwargs value is an extremely long string; boundary test."""
|
||||
request_body = {
|
||||
"model": "auto",
|
||||
"messages": [{"role": "user", "content": "你好"}],
|
||||
"chat_template_kwargs": {"custom_param": "a" * 10000}, # Extremely long string value
|
||||
"stream": stream,
|
||||
"max_tokens": 512,
|
||||
}
|
||||
|
||||
response = helper.send_request(api_client, "/v1/chat/completions", request_body)
|
||||
|
||||
# Check: status code should be 200 if the engine accepts it, or 400 if it exceeds the limit
|
||||
assert response.status_code in [
|
||||
200,
|
||||
400,
|
||||
], f"status code should be 200 or 400, got {response.status_code}"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("stream", [False, True], ids=["non_stream", "stream"])
|
||||
def test_chat_template_kwargs_many_keys(api_client, request, stream):
|
||||
"""chat_template_kwargs contains many key-value pairs; boundary test."""
|
||||
# Build an object with many keys
|
||||
kwargs = {f"param_{i}": f"value_{i}" for i in range(100)}
|
||||
|
||||
request_body = {
|
||||
"model": "auto",
|
||||
"messages": [{"role": "user", "content": "你好"}],
|
||||
"chat_template_kwargs": kwargs,
|
||||
"stream": stream,
|
||||
"max_tokens": 512,
|
||||
}
|
||||
|
||||
response = helper.send_request(api_client, "/v1/chat/completions", request_body)
|
||||
|
||||
# Check: status code should be 200 or 400
|
||||
assert response.status_code in [
|
||||
200,
|
||||
400,
|
||||
], f"status code should be 200 or 400, got {response.status_code}"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("stream", [False, True], ids=["non_stream", "stream"])
|
||||
def test_chat_template_kwargs_unicode_keys(api_client, request, stream):
|
||||
"""chat_template_kwargs contains Unicode key names; boundary test."""
|
||||
request_body = {
|
||||
"model": "auto",
|
||||
"messages": [{"role": "user", "content": "你好"}],
|
||||
"chat_template_kwargs": {
|
||||
"中文参数": "value",
|
||||
"日本語パラメータ": "value",
|
||||
"emoji_参数": "value",
|
||||
},
|
||||
"stream": stream,
|
||||
"max_tokens": 512,
|
||||
}
|
||||
|
||||
response = helper.send_request(api_client, "/v1/chat/completions", request_body)
|
||||
|
||||
# Check: status code should be 200 or 400 depending on whether the engine supports non-ASCII key names
|
||||
assert response.status_code in [
|
||||
200,
|
||||
400,
|
||||
], f"status code should be 200 or 400, got {response.status_code}"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("stream", [False, True], ids=["non_stream", "stream"])
|
||||
def test_chat_template_kwargs_special_chars_in_keys(api_client, request, stream):
|
||||
"""chat_template_kwargs key names contain special characters; boundary test."""
|
||||
request_body = {
|
||||
"model": "auto",
|
||||
"messages": [{"role": "user", "content": "你好"}],
|
||||
"chat_template_kwargs": {
|
||||
"param-with-dash": "value",
|
||||
"param_with_underscore": "value",
|
||||
"param.with.dot": "value",
|
||||
"param:with:colon": "value",
|
||||
},
|
||||
"stream": stream,
|
||||
"max_tokens": 512,
|
||||
}
|
||||
|
||||
response = helper.send_request(api_client, "/v1/chat/completions", request_body)
|
||||
|
||||
# Check: status code should be 200 or 400
|
||||
assert response.status_code in [
|
||||
200,
|
||||
400,
|
||||
], f"status code should be 200 or 400, got {response.status_code}"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("stream", [False, True], ids=["non_stream", "stream"])
|
||||
def test_chat_template_kwargs_numeric_string_values(api_client, request, stream):
|
||||
"""chat_template_kwargs values are numeric strings; boundary type-conversion test."""
|
||||
request_body = {
|
||||
"model": "auto",
|
||||
"messages": [{"role": "user", "content": "你好"}],
|
||||
"chat_template_kwargs": {
|
||||
"number_as_string": "12345",
|
||||
"float_as_string": "3.14159",
|
||||
"bool_as_string": "true",
|
||||
},
|
||||
"stream": stream,
|
||||
"max_tokens": 512,
|
||||
}
|
||||
|
||||
response = helper.send_request(api_client, "/v1/chat/completions", request_body)
|
||||
|
||||
# Check: status code should be 200
|
||||
assertion.assert_status_code_200(response)
|
||||
|
||||
# Check 2: streaming response contains [DONE]
|
||||
if stream:
|
||||
assertion.assert_stream_has_done(response.text)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("stream", [False, True], ids=["non_stream", "stream"])
|
||||
def test_chat_template_kwargs_mixed_types_values(api_client, request, stream):
|
||||
"""chat_template_kwargs values have mixed types (number, boolean, null); boundary test."""
|
||||
request_body = {
|
||||
"model": "auto",
|
||||
"messages": [{"role": "user", "content": "你好"}],
|
||||
"chat_template_kwargs": {
|
||||
"int_value": 42,
|
||||
"float_value": 3.14,
|
||||
"bool_value": True,
|
||||
"null_value": None,
|
||||
"string_value": "text",
|
||||
},
|
||||
"stream": stream,
|
||||
"max_tokens": 512,
|
||||
}
|
||||
|
||||
response = helper.send_request(api_client, "/v1/chat/completions", request_body)
|
||||
|
||||
# Check: status code should be 200 or 400 depending on how the engine handles non-string values
|
||||
assert response.status_code in [
|
||||
200,
|
||||
400,
|
||||
], f"status code should be 200 or 400, got {response.status_code}"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("stream", [False, True], ids=["non_stream", "stream"])
|
||||
def test_chat_template_kwargs_reserved_words_keys(api_client, request, stream):
|
||||
"""chat_template_kwargs uses reserved words or internal keywords as key names; boundary test."""
|
||||
request_body = {
|
||||
"model": "auto",
|
||||
"messages": [{"role": "user", "content": "你好"}],
|
||||
"chat_template_kwargs": {
|
||||
"model": "overridden_model", # Key that may conflict with request parameters
|
||||
"messages": "overridden", # Key that may conflict with request parameters
|
||||
"stream": True, # Key that may conflict with request parameters
|
||||
"temperature": 2.0, # Key that may conflict with generation parameters
|
||||
},
|
||||
"stream": stream,
|
||||
"max_tokens": 512,
|
||||
}
|
||||
|
||||
response = helper.send_request(api_client, "/v1/chat/completions", request_body)
|
||||
|
||||
# Check: status code should be 200 if the engine isolates namespaces correctly, or 400 if there is a conflict
|
||||
assert response.status_code in [
|
||||
200,
|
||||
400,
|
||||
], f"status code should be 200 or 400, got {response.status_code}"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("stream", [False, True], ids=["non_stream", "stream"])
|
||||
def test_chat_template_kwargs_empty_string_values(api_client, request, stream):
|
||||
"""chat_template_kwargs values are empty strings; boundary test."""
|
||||
request_body = {
|
||||
"model": "auto",
|
||||
"messages": [{"role": "user", "content": "你好"}],
|
||||
"chat_template_kwargs": {
|
||||
"empty_string": "",
|
||||
"whitespace_only": " ",
|
||||
"null_string": "null",
|
||||
},
|
||||
"stream": stream,
|
||||
"max_tokens": 512,
|
||||
}
|
||||
|
||||
response = helper.send_request(api_client, "/v1/chat/completions", request_body)
|
||||
|
||||
# Check 1: status code is 200
|
||||
assertion.assert_status_code_200(response)
|
||||
|
||||
# Check 2: streaming response contains [DONE]
|
||||
if stream:
|
||||
assertion.assert_stream_has_done(response.text)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("stream", [False, True], ids=["non_stream", "stream"])
|
||||
def test_chat_template_kwargs_deeply_nested_object(api_client, request, stream):
|
||||
"""chat_template_kwargs is a deeply nested object; boundary test."""
|
||||
request_body = {
|
||||
"model": "auto",
|
||||
"messages": [{"role": "user", "content": "你好"}],
|
||||
"chat_template_kwargs": {"level1": {"level2": {"level3": {"level4": {"level5": {"deep_value": "found"}}}}}},
|
||||
"stream": stream,
|
||||
"max_tokens": 512,
|
||||
}
|
||||
|
||||
response = helper.send_request(api_client, "/v1/chat/completions", request_body)
|
||||
|
||||
# Check: status code should be 200 if the engine flattens nested objects, or 400 if it rejects nesting
|
||||
assert response.status_code in [
|
||||
200,
|
||||
400,
|
||||
], f"status code should be 200 or 400, got {response.status_code}"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("stream", [False, True], ids=["non_stream", "stream"])
|
||||
def test_chat_template_kwargs_case_sensitive_keys(api_client, request, stream):
|
||||
"""chat_template_kwargs key names are case-sensitive; boundary test."""
|
||||
request_body = {
|
||||
"model": "auto",
|
||||
"messages": [{"role": "user", "content": "你好"}],
|
||||
"chat_template_kwargs": {
|
||||
"Add_Generation_Prompt": True, # Different from the standard snake_case form
|
||||
"ADD_GENERATION_PROMPT": True, # All uppercase
|
||||
"add_generation_prompt": True, # Standard lowercase
|
||||
},
|
||||
"stream": stream,
|
||||
"max_tokens": 512,
|
||||
}
|
||||
|
||||
response = helper.send_request(api_client, "/v1/chat/completions", request_body)
|
||||
|
||||
# Check: status code should be 200
|
||||
assertion.assert_status_code_200(response)
|
||||
|
||||
# Check 2: streaming response contains [DONE]
|
||||
if stream:
|
||||
assertion.assert_stream_has_done(response.text)
|
||||
@@ -0,0 +1,230 @@
|
||||
import pytest
|
||||
|
||||
from tests.e2e.weekly.single_node.engine_func_test_robot.utility import assertion
|
||||
from tests.e2e.weekly.single_node.engine_func_test_robot.utility import (
|
||||
request_helper as helper,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("stream", [False, True], ids=["non_stream", "stream"])
|
||||
def test_chat_template_kwargs_null(api_client, request, stream):
|
||||
"""chat_template_kwargs is null; the request should respond normally."""
|
||||
request_body = {
|
||||
"model": "auto",
|
||||
"messages": [{"role": "user", "content": "你好"}],
|
||||
"chat_template_kwargs": None,
|
||||
"stream": stream,
|
||||
"max_tokens": 512,
|
||||
}
|
||||
|
||||
response = helper.send_request(api_client, "/v1/chat/completions", request_body)
|
||||
|
||||
# Check 1: status code is 200
|
||||
assertion.assert_status_code_200(response)
|
||||
|
||||
# Check 2: streaming response contains [DONE]
|
||||
if stream:
|
||||
assertion.assert_stream_has_done(response.text)
|
||||
|
||||
# Check 3: finish_reason is stop or length
|
||||
if stream:
|
||||
finish_reason = assertion.assert_stream_single_finish_reason(response.text)
|
||||
else:
|
||||
finish_reason = response.json()["choices"][0]["finish_reason"]
|
||||
assertion.assert_finish_reason_valid(finish_reason)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("stream", [False, True], ids=["non_stream", "stream"])
|
||||
def test_chat_template_kwargs_empty_object(api_client, request, stream):
|
||||
"""chat_template_kwargs is an empty object {}; the optional field should be handled normally."""
|
||||
request_body = {
|
||||
"model": "auto",
|
||||
"messages": [{"role": "user", "content": "你好"}],
|
||||
"chat_template_kwargs": {},
|
||||
"stream": stream,
|
||||
"max_tokens": 512,
|
||||
}
|
||||
|
||||
response = helper.send_request(api_client, "/v1/chat/completions", request_body)
|
||||
|
||||
# Check 1: status code is 200
|
||||
assertion.assert_status_code_200(response)
|
||||
|
||||
# Check 2: streaming response contains [DONE]
|
||||
if stream:
|
||||
assertion.assert_stream_has_done(response.text)
|
||||
|
||||
# Check 3: finish_reason is valid
|
||||
if stream:
|
||||
finish_reason = assertion.assert_stream_single_finish_reason(response.text)
|
||||
else:
|
||||
finish_reason = response.json()["choices"][0]["finish_reason"]
|
||||
assertion.assert_finish_reason_valid(finish_reason)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("stream", [False, True], ids=["non_stream", "stream"])
|
||||
def test_chat_template_kwargs_with_add_generation_prompt(api_client, request, stream):
|
||||
"""Set add_generation_prompt in chat_template_kwargs to control generation prompt insertion."""
|
||||
request_body = {
|
||||
"model": "auto",
|
||||
"messages": [{"role": "user", "content": "你好"}],
|
||||
"chat_template_kwargs": {"add_generation_prompt": True},
|
||||
"stream": stream,
|
||||
"max_tokens": 512,
|
||||
}
|
||||
|
||||
response = helper.send_request(api_client, "/v1/chat/completions", request_body)
|
||||
|
||||
# Check 1: status code is 200
|
||||
assertion.assert_status_code_200(response)
|
||||
|
||||
# Check 2: streaming response contains [DONE]
|
||||
if stream:
|
||||
assertion.assert_stream_has_done(response.text)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("stream", [False, True], ids=["non_stream", "stream"])
|
||||
def test_chat_template_kwargs_with_custom_system_prompt(api_client, request, stream):
|
||||
"""Set custom system-prompt-related parameters in chat_template_kwargs."""
|
||||
request_body = {
|
||||
"model": "auto",
|
||||
"messages": [
|
||||
{"role": "system", "content": "你是AI助手"},
|
||||
{"role": "user", "content": "你好"},
|
||||
],
|
||||
"chat_template_kwargs": {"enable_system_prompt": True},
|
||||
"stream": stream,
|
||||
"max_tokens": 512,
|
||||
}
|
||||
|
||||
response = helper.send_request(api_client, "/v1/chat/completions", request_body)
|
||||
|
||||
# Check 1: status code is 200
|
||||
assertion.assert_status_code_200(response)
|
||||
|
||||
# Check 2: streaming response contains [DONE]
|
||||
if stream:
|
||||
assertion.assert_stream_has_done(response.text)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("stream", [False, True], ids=["non_stream", "stream"])
|
||||
def test_chat_template_kwargs_date_params(api_client, request, stream):
|
||||
"""chat_template_kwargs contains date-related parameters; some models support dynamic dates."""
|
||||
request_body = {
|
||||
"model": "auto",
|
||||
"messages": [{"role": "user", "content": "今天是星期几"}],
|
||||
"chat_template_kwargs": {"date": "2025-04-01", "time": "10:00:00"},
|
||||
"stream": stream,
|
||||
"max_tokens": 512,
|
||||
}
|
||||
|
||||
response = helper.send_request(api_client, "/v1/chat/completions", request_body)
|
||||
|
||||
# Check 1: status code is 200
|
||||
assertion.assert_status_code_200(response)
|
||||
|
||||
# Check 2: streaming response contains [DONE]
|
||||
if stream:
|
||||
assertion.assert_stream_has_done(response.text)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("stream", [False, True], ids=["non_stream", "stream"])
|
||||
def test_chat_template_kwargs_multiple_params(api_client, request, stream):
|
||||
"""chat_template_kwargs contains multiple valid parameters."""
|
||||
request_body = {
|
||||
"model": "auto",
|
||||
"messages": [{"role": "user", "content": "请简单回答"}],
|
||||
"chat_template_kwargs": {
|
||||
"add_generation_prompt": True,
|
||||
"tools_prompt": "default",
|
||||
"custom_var": "custom_value",
|
||||
},
|
||||
"stream": stream,
|
||||
"max_tokens": 512,
|
||||
}
|
||||
|
||||
response = helper.send_request(api_client, "/v1/chat/completions", request_body)
|
||||
|
||||
# Check 1: status code is 200
|
||||
assertion.assert_status_code_200(response)
|
||||
|
||||
# Check 2: streaming response contains [DONE]
|
||||
if stream:
|
||||
assertion.assert_stream_has_done(response.text)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("stream", [False, True], ids=["non_stream", "stream"])
|
||||
def test_chat_template_kwargs_with_tools_prompt(api_client, request, stream):
|
||||
"""Set tools_prompt in chat_template_kwargs to control the tool-call prompt format."""
|
||||
request_body = {
|
||||
"model": "auto",
|
||||
"messages": [{"role": "user", "content": "需要查询天气"}],
|
||||
"tools": [
|
||||
{
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "get_weather",
|
||||
"description": "获取天气信息",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {"location": {"type": "string"}},
|
||||
"required": ["location"],
|
||||
},
|
||||
},
|
||||
}
|
||||
],
|
||||
"chat_template_kwargs": {"tools_prompt": "tool_instruction"},
|
||||
"stream": stream,
|
||||
"max_tokens": 512,
|
||||
}
|
||||
|
||||
response = helper.send_request(api_client, "/v1/chat/completions", request_body)
|
||||
|
||||
# Check 1: status code is 200
|
||||
assertion.assert_status_code_200(response)
|
||||
|
||||
# Check 2: streaming response contains [DONE]
|
||||
if stream:
|
||||
assertion.assert_stream_has_done(response.text)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("stream", [False, True], ids=["non_stream", "stream"])
|
||||
def test_chat_template_kwargs_bos_token(api_client, request, stream):
|
||||
"""Set add_special_tokens or bos_token-related parameters in chat_template_kwargs."""
|
||||
request_body = {
|
||||
"model": "auto",
|
||||
"messages": [{"role": "user", "content": "你好"}],
|
||||
"chat_template_kwargs": {"add_special_tokens": True},
|
||||
"stream": stream,
|
||||
"max_tokens": 512,
|
||||
}
|
||||
|
||||
response = helper.send_request(api_client, "/v1/chat/completions", request_body)
|
||||
|
||||
# Check 1: status code is 200
|
||||
assertion.assert_status_code_200(response)
|
||||
|
||||
# Check 2: streaming response contains [DONE]
|
||||
if stream:
|
||||
assertion.assert_stream_has_done(response.text)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("stream", [False, True], ids=["non_stream", "stream"])
|
||||
def test_chat_template_kwargs_skip_special_tokens(api_client, request, stream):
|
||||
"""Set skip_special_tokens in chat_template_kwargs."""
|
||||
request_body = {
|
||||
"model": "auto",
|
||||
"messages": [{"role": "user", "content": "你好"}],
|
||||
"chat_template_kwargs": {"skip_special_tokens": False},
|
||||
"stream": stream,
|
||||
"max_tokens": 512,
|
||||
}
|
||||
|
||||
response = helper.send_request(api_client, "/v1/chat/completions", request_body)
|
||||
|
||||
# Check 1: status code is 200
|
||||
assertion.assert_status_code_200(response)
|
||||
|
||||
# Check 2: streaming response contains [DONE]
|
||||
if stream:
|
||||
assertion.assert_stream_has_done(response.text)
|
||||
@@ -0,0 +1,316 @@
|
||||
"""
|
||||
Tests for the thinking and enable_thinking fields.
|
||||
- thinking: used by DeepSeek/DS model families.
|
||||
- enable_thinking: used by Qwen model families.
|
||||
|
||||
When the field is true, validate that the think tags are complete.
|
||||
When the field is false, validate that no think tags are present.
|
||||
Follow the validation rules from the think_tag directory.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
|
||||
from tests.e2e.weekly.single_node.engine_func_test_robot.utility import assertion
|
||||
from tests.e2e.weekly.single_node.engine_func_test_robot.utility import (
|
||||
request_helper as helper,
|
||||
)
|
||||
|
||||
|
||||
def is_qwen_model(model_name):
|
||||
"""Return whether the model belongs to the Qwen family, case-insensitively."""
|
||||
return model_name and "qwen" in model_name.lower()
|
||||
|
||||
|
||||
def is_deepseek_model(model_name):
|
||||
"""Return whether the model belongs to the DeepSeek/DS family, case-insensitively."""
|
||||
if not model_name:
|
||||
return False
|
||||
model_lower = model_name.lower()
|
||||
return "deepseek" in model_lower or "ds" in model_lower
|
||||
|
||||
|
||||
def should_check_think_tag(request):
|
||||
"""Return whether think-tag validation should be performed."""
|
||||
return request.config.getoption("--thinkTagOutput").strip().lower() == "true"
|
||||
|
||||
|
||||
# ==================== Qwen Model Tests - enable_thinking ====================
|
||||
|
||||
|
||||
@pytest.mark.parametrize("stream", [False, True], ids=["non_stream", "stream"])
|
||||
def test_chat_template_kwargs_qwen_enable_thinking_true(api_client, request, stream):
|
||||
"""Qwen model: enable_thinking=true enables thinking mode; validate complete think tags."""
|
||||
model = request.config.getoption("--model")
|
||||
if not is_qwen_model(model):
|
||||
pytest.skip(f"current model {model} is not in the Qwen family; skipping this test")
|
||||
|
||||
request_body = {
|
||||
"model": "auto",
|
||||
"messages": [{"role": "user", "content": "请用最简单的一句话介绍你是谁。"}],
|
||||
"chat_template_kwargs": {"enable_thinking": True},
|
||||
"stream": stream,
|
||||
"max_tokens": 512,
|
||||
}
|
||||
|
||||
response = helper.send_request(api_client, "/v1/chat/completions", request_body)
|
||||
|
||||
# Check 1: status code is 200
|
||||
assertion.assert_status_code_200(response)
|
||||
|
||||
# Check 2: streaming response contains [DONE]
|
||||
if stream:
|
||||
assertion.assert_stream_has_done(response.text)
|
||||
|
||||
# Check 3: think tags are complete when enable_thinking=true
|
||||
if should_check_think_tag(request):
|
||||
assertion.assert_think_tag_present(response.content.decode("utf-8"), "enable_thinking=true")
|
||||
|
||||
|
||||
@pytest.mark.parametrize("stream", [False, True], ids=["non_stream", "stream"])
|
||||
def test_chat_template_kwargs_qwen_enable_thinking_false(api_client, request, stream):
|
||||
"""Qwen model: enable_thinking=false disables thinking mode; validate that no think tags are present."""
|
||||
model = request.config.getoption("--model")
|
||||
if not is_qwen_model(model):
|
||||
pytest.skip(f"current model {model} is not in the Qwen family; skipping this test")
|
||||
|
||||
request_body = {
|
||||
"model": "auto",
|
||||
"messages": [{"role": "user", "content": "请用最简单的一句话介绍你是谁。"}],
|
||||
"chat_template_kwargs": {"enable_thinking": False},
|
||||
"stream": stream,
|
||||
"max_tokens": 512,
|
||||
}
|
||||
|
||||
response = helper.send_request(api_client, "/v1/chat/completions", request_body)
|
||||
|
||||
# Check 1: status code is 200
|
||||
assertion.assert_status_code_200(response)
|
||||
|
||||
# Check 2: streaming response contains [DONE]
|
||||
if stream:
|
||||
assertion.assert_stream_has_done(response.text)
|
||||
|
||||
# Check 3: no think tags are present when enable_thinking=false
|
||||
if should_check_think_tag(request):
|
||||
assertion.assert_no_think_tag(response.content.decode("utf-8"), "enable_thinking=false")
|
||||
|
||||
|
||||
# ==================== DeepSeek/DS Model Tests - thinking ====================
|
||||
|
||||
|
||||
@pytest.mark.parametrize("stream", [False, True], ids=["non_stream", "stream"])
|
||||
def test_chat_template_kwargs_deepseek_thinking_true(api_client, request, stream):
|
||||
"""DeepSeek/DS model: thinking=true enables thinking mode; validate complete think tags."""
|
||||
model = request.config.getoption("--model")
|
||||
if not is_deepseek_model(model):
|
||||
pytest.skip(f"current model {model} is not in the DeepSeek/DS family; skipping this test")
|
||||
|
||||
request_body = {
|
||||
"model": "auto",
|
||||
"messages": [{"role": "user", "content": "请用最简单的一句话介绍你是谁。"}],
|
||||
"chat_template_kwargs": {"thinking": True},
|
||||
"stream": stream,
|
||||
"max_tokens": 512,
|
||||
}
|
||||
|
||||
response = helper.send_request(api_client, "/v1/chat/completions", request_body)
|
||||
|
||||
# Check 1: status code is 200
|
||||
assertion.assert_status_code_200(response)
|
||||
|
||||
# Check 2: streaming response contains [DONE]
|
||||
if stream:
|
||||
assertion.assert_stream_has_done(response.text)
|
||||
|
||||
# Check 3: think tags are complete when thinking=true
|
||||
if should_check_think_tag(request):
|
||||
assertion.assert_think_tag_present(response.content.decode("utf-8"), "thinking=true")
|
||||
|
||||
|
||||
@pytest.mark.parametrize("stream", [False, True], ids=["non_stream", "stream"])
|
||||
def test_chat_template_kwargs_deepseek_thinking_false(api_client, request, stream):
|
||||
"""DeepSeek/DS model: thinking=false disables thinking mode; validate that no think tags are present."""
|
||||
model = request.config.getoption("--model")
|
||||
if not is_deepseek_model(model):
|
||||
pytest.skip(f"current model {model} is not in the DeepSeek/DS family; skipping this test")
|
||||
|
||||
request_body = {
|
||||
"model": "auto",
|
||||
"messages": [{"role": "user", "content": "请用最简单的一句话介绍你是谁。"}],
|
||||
"chat_template_kwargs": {"thinking": False},
|
||||
"stream": stream,
|
||||
"max_tokens": 512,
|
||||
}
|
||||
|
||||
response = helper.send_request(api_client, "/v1/chat/completions", request_body)
|
||||
|
||||
# Check 1: status code is 200
|
||||
assertion.assert_status_code_200(response)
|
||||
|
||||
# Check 2: streaming response contains [DONE]
|
||||
if stream:
|
||||
assertion.assert_stream_has_done(response.text)
|
||||
|
||||
# Check 3: no think tags are present when thinking=false
|
||||
if should_check_think_tag(request):
|
||||
assertion.assert_no_think_tag(response.content.decode("utf-8"), "thinking=false")
|
||||
|
||||
|
||||
# ==================== Inapplicable Model Tests - Abnormal Scenarios ====================
|
||||
|
||||
|
||||
@pytest.mark.parametrize("stream", [False, True], ids=["non_stream", "stream"])
|
||||
def test_chat_template_kwargs_qwen_field_on_deepseek(api_client, request, stream):
|
||||
"""Abnormal: use the enable_thinking field on a DeepSeek model."""
|
||||
model = request.config.getoption("--model")
|
||||
if not is_deepseek_model(model):
|
||||
pytest.skip(f"current model {model} is not in the DeepSeek/DS family; skipping this test")
|
||||
|
||||
request_body = {
|
||||
"model": "auto",
|
||||
"messages": [{"role": "user", "content": "你好"}],
|
||||
"chat_template_kwargs": {"enable_thinking": True}, # Use the Qwen field on DeepSeek
|
||||
"stream": stream,
|
||||
"max_tokens": 512,
|
||||
}
|
||||
|
||||
response = helper.send_request(api_client, "/v1/chat/completions", request_body)
|
||||
|
||||
# Check: either 200 if the engine ignores unknown fields, or 400 if it validates strictly
|
||||
assert response.status_code in [
|
||||
200,
|
||||
400,
|
||||
], f"status code should be 200 or 400, got {response.status_code}"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("stream", [False, True], ids=["non_stream", "stream"])
|
||||
def test_chat_template_kwargs_deepseek_field_on_qwen(api_client, request, stream):
|
||||
"""Abnormal: use the thinking field on a Qwen model."""
|
||||
model = request.config.getoption("--model")
|
||||
if not is_qwen_model(model):
|
||||
pytest.skip(f"current model {model} is not in the Qwen family; skipping this test")
|
||||
|
||||
request_body = {
|
||||
"model": "auto",
|
||||
"messages": [{"role": "user", "content": "你好"}],
|
||||
"chat_template_kwargs": {"thinking": True}, # Use the DeepSeek field on Qwen
|
||||
"stream": stream,
|
||||
"max_tokens": 512,
|
||||
}
|
||||
|
||||
response = helper.send_request(api_client, "/v1/chat/completions", request_body)
|
||||
|
||||
# Check: either 200 if the engine ignores unknown fields, or 400 if it validates strictly
|
||||
assert response.status_code in [
|
||||
200,
|
||||
400,
|
||||
], f"status code should be 200 or 400, got {response.status_code}"
|
||||
|
||||
|
||||
# ==================== Boundary Tests ====================
|
||||
|
||||
|
||||
@pytest.mark.parametrize("stream", [False, True], ids=["non_stream", "stream"])
|
||||
def test_chat_template_kwargs_qwen_enable_thinking_null(api_client, request, stream):
|
||||
"""Abnormal: enable_thinking is null for a Qwen model."""
|
||||
model = request.config.getoption("--model")
|
||||
if not is_qwen_model(model):
|
||||
pytest.skip(f"current model {model} is not in the Qwen family; skipping this test")
|
||||
|
||||
request_body = {
|
||||
"model": "auto",
|
||||
"messages": [{"role": "user", "content": "你好"}],
|
||||
"chat_template_kwargs": {"enable_thinking": None},
|
||||
"stream": stream,
|
||||
"max_tokens": 512,
|
||||
}
|
||||
|
||||
response = helper.send_request(api_client, "/v1/chat/completions", request_body)
|
||||
|
||||
assertion.assert_status_code_200(response)
|
||||
|
||||
|
||||
def test_chat_template_kwargs_deepseek_thinking_null_non_stream(api_client, request):
|
||||
"""Abnormal: thinking is null for a DeepSeek model in non-streaming mode; error code is 400."""
|
||||
model = request.config.getoption("--model")
|
||||
if not is_deepseek_model(model):
|
||||
pytest.skip(f"current model {model} is not in the DeepSeek/DS family; skipping this test")
|
||||
|
||||
request_body = {
|
||||
"model": "auto",
|
||||
"messages": [{"role": "user", "content": "你好"}],
|
||||
"chat_template_kwargs": {"thinking": None},
|
||||
"stream": False,
|
||||
"max_tokens": 512,
|
||||
}
|
||||
|
||||
response = helper.send_request(api_client, "/v1/chat/completions", request_body)
|
||||
|
||||
assertion.assert_status_code_200(response)
|
||||
|
||||
|
||||
def test_chat_template_kwargs_deepseek_thinking_null_stream(api_client, request):
|
||||
"""Abnormal: thinking is null for a DeepSeek model in streaming mode; status code is 200 and error code is 400."""
|
||||
model = request.config.getoption("--model")
|
||||
if not is_deepseek_model(model):
|
||||
pytest.skip(f"current model {model} is not in the DeepSeek/DS family; skipping this test")
|
||||
|
||||
request_body = {
|
||||
"model": "auto",
|
||||
"messages": [{"role": "user", "content": "你好"}],
|
||||
"chat_template_kwargs": {"thinking": None},
|
||||
"stream": True,
|
||||
"max_tokens": 512,
|
||||
}
|
||||
|
||||
response = helper.send_request(api_client, "/v1/chat/completions", request_body)
|
||||
|
||||
# Check: status code is 200 and error code is 400
|
||||
assertion.assert_status_code_200(response)
|
||||
assertion.assert_error_code_400(response)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("stream", [False, True], ids=["non_stream", "stream"])
|
||||
def test_chat_template_kwargs_qwen_enable_thinking_string(api_client, request, stream):
|
||||
"""Abnormal: enable_thinking is a string for a Qwen model."""
|
||||
model = request.config.getoption("--model")
|
||||
if not is_qwen_model(model):
|
||||
pytest.skip(f"current model {model} is not in the Qwen family; skipping this test")
|
||||
|
||||
request_body = {
|
||||
"model": "auto",
|
||||
"messages": [{"role": "user", "content": "你好"}],
|
||||
"chat_template_kwargs": {"enable_thinking": "true"},
|
||||
"stream": stream,
|
||||
"max_tokens": 512,
|
||||
}
|
||||
|
||||
response = helper.send_request(api_client, "/v1/chat/completions", request_body)
|
||||
|
||||
assert response.status_code in [
|
||||
200,
|
||||
400,
|
||||
], f"status code should be 200 or 400, got {response.status_code}"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("stream", [False, True], ids=["non_stream", "stream"])
|
||||
def test_chat_template_kwargs_deepseek_thinking_string(api_client, request, stream):
|
||||
"""Abnormal: thinking is a string for a DeepSeek model."""
|
||||
model = request.config.getoption("--model")
|
||||
if not is_deepseek_model(model):
|
||||
pytest.skip(f"current model {model} is not in the DeepSeek/DS family; skipping this test")
|
||||
|
||||
request_body = {
|
||||
"model": "auto",
|
||||
"messages": [{"role": "user", "content": "你好"}],
|
||||
"chat_template_kwargs": {"thinking": "true"},
|
||||
"stream": stream,
|
||||
"max_tokens": 512,
|
||||
}
|
||||
|
||||
response = helper.send_request(api_client, "/v1/chat/completions", request_body)
|
||||
|
||||
assert response.status_code in [
|
||||
200,
|
||||
400,
|
||||
], f"status code should be 200 or 400, got {response.status_code}"
|
||||
@@ -0,0 +1 @@
|
||||
# Content field test package
|
||||
@@ -0,0 +1,264 @@
|
||||
from tests.e2e.weekly.single_node.engine_func_test_robot.utility import assertion
|
||||
from tests.e2e.weekly.single_node.engine_func_test_robot.utility import (
|
||||
request_helper as helper,
|
||||
)
|
||||
|
||||
|
||||
def test_content_integer_non_stream(api_client, request):
|
||||
"""Non-streaming: content is integer type, should return 400 error"""
|
||||
request_body = {
|
||||
"model": "auto",
|
||||
"messages": [{"role": "user", "content": 12345}],
|
||||
"stream": False,
|
||||
"max_tokens": 512,
|
||||
}
|
||||
|
||||
response = helper.send_request(api_client, "/v1/chat/completions", request_body)
|
||||
|
||||
# Checkpoint: status code 400, error code 400
|
||||
assertion.assert_status_code_400(response)
|
||||
assertion.assert_error_code_400(response)
|
||||
|
||||
|
||||
def test_content_integer_stream(api_client, request):
|
||||
"""Streaming: content is integer type, should return 400 error"""
|
||||
request_body = {
|
||||
"model": "auto",
|
||||
"messages": [{"role": "user", "content": 12345}],
|
||||
"stream": True,
|
||||
"max_tokens": 512,
|
||||
}
|
||||
|
||||
response = helper.send_request(api_client, "/v1/chat/completions", request_body)
|
||||
|
||||
# Checkpoint: status code 400, error code 400
|
||||
assertion.assert_status_code_400(response)
|
||||
assertion.assert_error_code_400(response)
|
||||
|
||||
|
||||
def test_content_object_non_stream(api_client, request):
|
||||
"""Non-streaming: content is object type (non-standard multimodal format), should return 400 error"""
|
||||
request_body = {
|
||||
"model": "auto",
|
||||
"messages": [{"role": "user", "content": {"text": "hello", "extra": "data"}}],
|
||||
"stream": False,
|
||||
"max_tokens": 512,
|
||||
}
|
||||
|
||||
response = helper.send_request(api_client, "/v1/chat/completions", request_body)
|
||||
|
||||
# Checkpoint: status code 400, error code 400
|
||||
assertion.assert_status_code_400(response)
|
||||
assertion.assert_error_code_400(response)
|
||||
|
||||
|
||||
def test_content_object_stream(api_client, request):
|
||||
"""Streaming: content is object type, should return 400 error"""
|
||||
request_body = {
|
||||
"model": "auto",
|
||||
"messages": [{"role": "user", "content": {"text": "hello", "extra": "data"}}],
|
||||
"stream": True,
|
||||
"max_tokens": 512,
|
||||
}
|
||||
|
||||
response = helper.send_request(api_client, "/v1/chat/completions", request_body)
|
||||
|
||||
# Checkpoint: status code 400, error code 400
|
||||
assertion.assert_status_code_400(response)
|
||||
assertion.assert_error_code_400(response)
|
||||
|
||||
|
||||
def test_content_boolean_non_stream(api_client, request):
|
||||
"""Non-streaming: content is boolean type, should return 400 error"""
|
||||
request_body = {
|
||||
"model": "auto",
|
||||
"messages": [{"role": "user", "content": True}],
|
||||
"stream": False,
|
||||
"max_tokens": 512,
|
||||
}
|
||||
|
||||
response = helper.send_request(api_client, "/v1/chat/completions", request_body)
|
||||
|
||||
# Checkpoint: status code 400, error code 400
|
||||
assertion.assert_status_code_400(response)
|
||||
assertion.assert_error_code_400(response)
|
||||
|
||||
|
||||
def test_content_boolean_stream(api_client, request):
|
||||
"""Streaming: content is boolean type, should return 400 error"""
|
||||
request_body = {
|
||||
"model": "auto",
|
||||
"messages": [{"role": "user", "content": False}],
|
||||
"stream": True,
|
||||
"max_tokens": 512,
|
||||
}
|
||||
|
||||
response = helper.send_request(api_client, "/v1/chat/completions", request_body)
|
||||
|
||||
# Checkpoint: status code 400, error code 400
|
||||
assertion.assert_status_code_400(response)
|
||||
assertion.assert_error_code_400(response)
|
||||
|
||||
|
||||
def test_content_array_invalid_format_non_stream(api_client, request):
|
||||
"""Non-streaming: content is array but format does not conform to OpenAI multimodal spec (string array),
|
||||
should return 400 error"""
|
||||
request_body = {
|
||||
"model": "auto",
|
||||
"messages": [{"role": "user", "content": ["invalid", "array", "format"]}],
|
||||
"stream": False,
|
||||
"max_tokens": 512,
|
||||
}
|
||||
|
||||
response = helper.send_request(api_client, "/v1/chat/completions", request_body)
|
||||
|
||||
# Checkpoint: status code 400, error code 400
|
||||
assertion.assert_status_code_400(response)
|
||||
assertion.assert_error_code_400(response)
|
||||
|
||||
|
||||
def test_content_array_invalid_format_stream(api_client, request):
|
||||
"""Streaming: content is array but format does not conform to OpenAI multimodal spec, should return 400 error"""
|
||||
request_body = {
|
||||
"model": "auto",
|
||||
"messages": [{"role": "user", "content": ["invalid", "array", "format"]}],
|
||||
"stream": True,
|
||||
"max_tokens": 512,
|
||||
}
|
||||
|
||||
response = helper.send_request(api_client, "/v1/chat/completions", request_body)
|
||||
|
||||
# Checkpoint: status code 400, error code 400
|
||||
assertion.assert_status_code_400(response)
|
||||
assertion.assert_error_code_400(response)
|
||||
|
||||
|
||||
# ==================== Content Array Abnormal Tests ====================
|
||||
|
||||
|
||||
def test_content_array_missing_type_non_stream(api_client, request):
|
||||
"""Non-streaming: content array object missing type field, should return 400 error"""
|
||||
request_body = {
|
||||
"model": "auto",
|
||||
"messages": [{"role": "user", "content": [{"text": "你好"}]}], # missing type field
|
||||
"stream": False,
|
||||
"max_tokens": 512,
|
||||
}
|
||||
|
||||
response = helper.send_request(api_client, "/v1/chat/completions", request_body)
|
||||
|
||||
# Checkpoint: status code 400, error code 400
|
||||
assertion.assert_status_code_400(response)
|
||||
assertion.assert_error_code_400(response)
|
||||
|
||||
|
||||
def test_content_array_missing_type_stream(api_client, request):
|
||||
"""Streaming: content array object missing type field, should return 400 error"""
|
||||
request_body = {
|
||||
"model": "auto",
|
||||
"messages": [{"role": "user", "content": [{"text": "你好"}]}],
|
||||
"stream": True,
|
||||
"max_tokens": 512,
|
||||
}
|
||||
|
||||
response = helper.send_request(api_client, "/v1/chat/completions", request_body)
|
||||
|
||||
# Checkpoint: status code 400, error code 400
|
||||
assertion.assert_status_code_400(response)
|
||||
assertion.assert_error_code_400(response)
|
||||
|
||||
|
||||
def test_content_array_missing_text_non_stream(api_client, request):
|
||||
"""Non-streaming: content array object type is text but missing text field, should return 400 error"""
|
||||
request_body = {
|
||||
"model": "auto",
|
||||
"messages": [{"role": "user", "content": [{"type": "text"}]}], # missing text field
|
||||
"stream": False,
|
||||
"max_tokens": 512,
|
||||
}
|
||||
|
||||
response = helper.send_request(api_client, "/v1/chat/completions", request_body)
|
||||
|
||||
# Checkpoint: status code 400, error code 400
|
||||
assertion.assert_status_code_400(response)
|
||||
assertion.assert_error_code_400(response)
|
||||
|
||||
|
||||
def test_content_array_invalid_type_non_stream(api_client, request):
|
||||
"""Non-streaming: content array object type is invalid, should return 400 error"""
|
||||
request_body = {
|
||||
"model": "auto",
|
||||
"messages": [{"role": "user", "content": [{"type": "invalid_type", "text": "你好"}]}],
|
||||
"stream": False,
|
||||
"max_tokens": 512,
|
||||
}
|
||||
|
||||
response = helper.send_request(api_client, "/v1/chat/completions", request_body)
|
||||
|
||||
# Checkpoint: status code 400, error code 400
|
||||
assertion.assert_status_code_400(response)
|
||||
assertion.assert_error_code_400(response)
|
||||
|
||||
|
||||
def test_content_array_invalid_type_stream(api_client, request):
|
||||
"""Streaming: content array object type is invalid, should return 400 error"""
|
||||
request_body = {
|
||||
"model": "auto",
|
||||
"messages": [{"role": "user", "content": [{"type": "unknown", "text": "你好"}]}],
|
||||
"stream": True,
|
||||
"max_tokens": 512,
|
||||
}
|
||||
|
||||
response = helper.send_request(api_client, "/v1/chat/completions", request_body)
|
||||
|
||||
# Checkpoint: status code 400, error code 400
|
||||
assertion.assert_status_code_400(response)
|
||||
assertion.assert_error_code_400(response)
|
||||
|
||||
|
||||
def test_content_array_text_field_null_non_stream(api_client, request):
|
||||
"""Non-streaming: content array object text field is null, should return 400 error"""
|
||||
request_body = {
|
||||
"model": "auto",
|
||||
"messages": [{"role": "user", "content": [{"type": "text", "text": None}]}],
|
||||
"stream": False,
|
||||
"max_tokens": 512,
|
||||
}
|
||||
|
||||
response = helper.send_request(api_client, "/v1/chat/completions", request_body)
|
||||
|
||||
# Checkpoint: status code 400, error code 400
|
||||
assertion.assert_status_code_400(response)
|
||||
assertion.assert_error_code_400(response)
|
||||
|
||||
|
||||
def test_content_array_text_field_integer_non_stream(api_client, request):
|
||||
"""Non-streaming: content array object text field is integer type, should return 400 error"""
|
||||
request_body = {
|
||||
"model": "auto",
|
||||
"messages": [{"role": "user", "content": [{"type": "text", "text": 12345}]}],
|
||||
"stream": False,
|
||||
"max_tokens": 512,
|
||||
}
|
||||
|
||||
response = helper.send_request(api_client, "/v1/chat/completions", request_body)
|
||||
|
||||
# Checkpoint: status code 400, error code 400
|
||||
assertion.assert_status_code_400(response)
|
||||
assertion.assert_error_code_400(response)
|
||||
|
||||
|
||||
def test_content_array_text_field_integer_stream(api_client, request):
|
||||
"""Streaming: content array object text field is integer type, should return 400 error"""
|
||||
request_body = {
|
||||
"model": "auto",
|
||||
"messages": [{"role": "user", "content": [{"type": "text", "text": 12345}]}],
|
||||
"stream": True,
|
||||
"max_tokens": 512,
|
||||
}
|
||||
|
||||
response = helper.send_request(api_client, "/v1/chat/completions", request_body)
|
||||
|
||||
# Checkpoint: status code 400, error code 400
|
||||
assertion.assert_status_code_400(response)
|
||||
assertion.assert_error_code_400(response)
|
||||
@@ -0,0 +1,307 @@
|
||||
import pytest
|
||||
|
||||
from tests.e2e.weekly.single_node.engine_func_test_robot.utility import assertion
|
||||
from tests.e2e.weekly.single_node.engine_func_test_robot.utility import (
|
||||
request_helper as helper,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("stream", [False, True], ids=["non_stream", "stream"])
|
||||
def test_content_whitespace_only(api_client, request, stream):
|
||||
"""Content contains only whitespace characters (spaces, tabs, newlines), boundary case handling"""
|
||||
request_body = {
|
||||
"model": "auto",
|
||||
"messages": [{"role": "user", "content": " \t\n\n "}],
|
||||
"stream": stream,
|
||||
"max_tokens": 512,
|
||||
}
|
||||
|
||||
response = helper.send_request(api_client, "/v1/chat/completions", request_body)
|
||||
|
||||
# Checkpoint: status code should be 200 or 400 (depends on engine implementation)
|
||||
assert response.status_code in [
|
||||
200,
|
||||
400,
|
||||
], f"Status code should be 200 or 400, got {response.status_code}"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("stream", [False, True], ids=["non_stream", "stream"])
|
||||
def test_content_single_char(api_client, request, stream):
|
||||
"""Content is a single character, minimum valid content boundary"""
|
||||
request_body = {
|
||||
"model": "auto",
|
||||
"messages": [{"role": "user", "content": "?"}],
|
||||
"stream": stream,
|
||||
"max_tokens": 512,
|
||||
}
|
||||
|
||||
response = helper.send_request(api_client, "/v1/chat/completions", request_body)
|
||||
|
||||
# Checkpoint 1: status code 200
|
||||
assertion.assert_status_code_200(response)
|
||||
|
||||
# Checkpoint 2: streaming response contains [DONE]
|
||||
if stream:
|
||||
assertion.assert_stream_has_done(response.text)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("stream", [False, True], ids=["non_stream", "stream"])
|
||||
def test_content_with_null_bytes(api_client, request, stream):
|
||||
"""Content contains null bytes \x00, boundary security test"""
|
||||
request_body = {
|
||||
"model": "auto",
|
||||
"messages": [{"role": "user", "content": "Hello\x00World"}],
|
||||
"stream": stream,
|
||||
"max_tokens": 512,
|
||||
}
|
||||
|
||||
response = helper.send_request(api_client, "/v1/chat/completions", request_body)
|
||||
|
||||
# Checkpoint: status code should be 200 or 400 (depends on how engine handles null bytes)
|
||||
assert response.status_code in [
|
||||
200,
|
||||
400,
|
||||
], f"Status code should be 200 or 400, got {response.status_code}"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("stream", [False, True], ids=["non_stream", "stream"])
|
||||
def test_content_json_escape_sequences(api_client, request, stream):
|
||||
"""Content contains JSON escape characters, boundary test"""
|
||||
request_body = {
|
||||
"model": "auto",
|
||||
"messages": [{"role": "user", "content": 'Line1\nLine2\tTabbed"Quoted"\\Backslash'}],
|
||||
"stream": stream,
|
||||
"max_tokens": 512,
|
||||
}
|
||||
|
||||
response = helper.send_request(api_client, "/v1/chat/completions", request_body)
|
||||
|
||||
# Checkpoint 1: status code 200
|
||||
assertion.assert_status_code_200(response)
|
||||
|
||||
# Checkpoint 2: streaming response contains [DONE]
|
||||
if stream:
|
||||
assertion.assert_stream_has_done(response.text)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("stream", [False, True], ids=["non_stream", "stream"])
|
||||
def test_content_unicode_edge_cases(api_client, request, stream):
|
||||
"""Content contains Unicode boundary characters (e.g., zero-width characters, combining characters)"""
|
||||
request_body = {
|
||||
"model": "auto",
|
||||
"messages": [
|
||||
{
|
||||
"role": "user",
|
||||
"content": "零宽空格:\u200b 零宽连接符:\u200d 从右向左符:\u202e 组合字符:é",
|
||||
}
|
||||
],
|
||||
"stream": stream,
|
||||
"max_tokens": 512,
|
||||
}
|
||||
|
||||
response = helper.send_request(api_client, "/v1/chat/completions", request_body)
|
||||
|
||||
# Checkpoint 1: status code 200
|
||||
assertion.assert_status_code_200(response)
|
||||
|
||||
# Checkpoint 2: streaming response contains [DONE]
|
||||
if stream:
|
||||
assertion.assert_stream_has_done(response.text)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("stream", [False, True], ids=["non_stream", "stream"])
|
||||
def test_content_rare_unicode_blocks(api_client, request, stream):
|
||||
"""Content contains rare Unicode block characters (emoji variants, math symbols, etc.)"""
|
||||
request_body = {
|
||||
"model": "auto",
|
||||
"messages": [
|
||||
{
|
||||
"role": "user",
|
||||
"content": "数学:∀∃∈∉ 表情变体:👨🏻💻 盲文:⠓⠑⠇⠇⠕ 箭头:↳↴↵",
|
||||
}
|
||||
],
|
||||
"stream": stream,
|
||||
"max_tokens": 512,
|
||||
}
|
||||
|
||||
response = helper.send_request(api_client, "/v1/chat/completions", request_body)
|
||||
|
||||
# Checkpoint 1: status code 200
|
||||
assertion.assert_status_code_200(response)
|
||||
|
||||
# Checkpoint 2: streaming response contains [DONE]
|
||||
if stream:
|
||||
assertion.assert_stream_has_done(response.text)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("stream", [False, True], ids=["non_stream", "stream"])
|
||||
def test_content_rtl_languages(api_client, request, stream):
|
||||
"""Content contains right-to-left languages (Arabic, Hebrew, etc.)"""
|
||||
request_body = {
|
||||
"model": "auto",
|
||||
"messages": [{"role": "user", "content": "مرحبا بالعالم (Arabic) שלום עולם (Hebrew)"}],
|
||||
"stream": stream,
|
||||
"max_tokens": 512,
|
||||
}
|
||||
|
||||
response = helper.send_request(api_client, "/v1/chat/completions", request_body)
|
||||
|
||||
# Checkpoint 1: status code 200
|
||||
assertion.assert_status_code_200(response)
|
||||
|
||||
# Checkpoint 2: streaming response contains [DONE]
|
||||
if stream:
|
||||
assertion.assert_stream_has_done(response.text)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("stream", [False, True], ids=["non_stream", "stream"])
|
||||
def test_content_mixed_encoding_simulation(api_client, request, stream):
|
||||
"""Content simulates mixed encoding scenario (correctly encoded UTF-8)"""
|
||||
request_body = {
|
||||
"model": "auto",
|
||||
"messages": [{"role": "user", "content": "Mixed: English中文العربية日本語🌍"}],
|
||||
"stream": stream,
|
||||
"max_tokens": 512,
|
||||
}
|
||||
|
||||
response = helper.send_request(api_client, "/v1/chat/completions", request_body)
|
||||
|
||||
# Checkpoint 1: status code 200
|
||||
assertion.assert_status_code_200(response)
|
||||
|
||||
# Checkpoint 2: streaming response contains [DONE]
|
||||
if stream:
|
||||
assertion.assert_stream_has_done(response.text)
|
||||
|
||||
|
||||
# ==================== Content Array Format Boundary Tests ====================
|
||||
|
||||
|
||||
@pytest.mark.parametrize("stream", [False, True], ids=["non_stream", "stream"])
|
||||
def test_content_array_type_field_case_sensitive(api_client, request, stream):
|
||||
"""Content array format type field case sensitivity boundary test"""
|
||||
request_body = {
|
||||
"model": "auto",
|
||||
"messages": [{"role": "user", "content": [{"type": "TEXT", "text": "你好"}]}], # uppercase TEXT
|
||||
"stream": stream,
|
||||
"max_tokens": 512,
|
||||
}
|
||||
|
||||
response = helper.send_request(api_client, "/v1/chat/completions", request_body)
|
||||
|
||||
# Checkpoint: status code should be 200 (insensitive) or 400 (sensitive)
|
||||
assert response.status_code in [
|
||||
200,
|
||||
400,
|
||||
], f"Status code should be 200 or 400, got {response.status_code}"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("stream", [False, True], ids=["non_stream", "stream"])
|
||||
def test_content_array_extra_fields(api_client, request, stream):
|
||||
"""Content array format contains extra fields, boundary test"""
|
||||
request_body = {
|
||||
"model": "auto",
|
||||
"messages": [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [{"type": "text", "text": "你好", "extra_field": "extra_value"}],
|
||||
}
|
||||
],
|
||||
"stream": stream,
|
||||
"max_tokens": 512,
|
||||
}
|
||||
|
||||
response = helper.send_request(api_client, "/v1/chat/completions", request_body)
|
||||
|
||||
# Checkpoint: status code should be 200 (if engine ignores extra fields) or 400 (if strict validation)
|
||||
assert response.status_code in [
|
||||
200,
|
||||
400,
|
||||
], f"Status code should be 200 or 400, got {response.status_code}"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("stream", [False, True], ids=["non_stream", "stream"])
|
||||
def test_content_array_text_whitespace_only(api_client, request, stream):
|
||||
"""Content array format text field contains only whitespace characters, boundary test"""
|
||||
request_body = {
|
||||
"model": "auto",
|
||||
"messages": [{"role": "user", "content": [{"type": "text", "text": " \t\n "}]}],
|
||||
"stream": stream,
|
||||
"max_tokens": 512,
|
||||
}
|
||||
|
||||
response = helper.send_request(api_client, "/v1/chat/completions", request_body)
|
||||
|
||||
# Checkpoint: status code should be 200 or 400 (depends on engine implementation)
|
||||
assert response.status_code in [
|
||||
200,
|
||||
400,
|
||||
], f"Status code should be 200 or 400, got {response.status_code}"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("stream", [False, True], ids=["non_stream", "stream"])
|
||||
def test_content_array_very_long_text(api_client, request, stream):
|
||||
"""Content array format text field is extremely long text, boundary test"""
|
||||
request_body = {
|
||||
"model": "auto",
|
||||
"messages": [{"role": "user", "content": [{"type": "text", "text": "A" * 5000}]}],
|
||||
"stream": stream,
|
||||
"max_tokens": 512,
|
||||
}
|
||||
|
||||
response = helper.send_request(api_client, "/v1/chat/completions", request_body)
|
||||
|
||||
# Checkpoint: status code should be 200 or 400
|
||||
assert response.status_code in [
|
||||
200,
|
||||
400,
|
||||
], f"Status code should be 200 or 400, got {response.status_code}"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("stream", [False, True], ids=["non_stream", "stream"])
|
||||
def test_content_array_many_objects(api_client, request, stream):
|
||||
"""Content array format contains many text objects, boundary test"""
|
||||
request_body = {
|
||||
"model": "auto",
|
||||
"messages": [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [{"type": "text", "text": f"分段{i}"} for i in range(50)],
|
||||
}
|
||||
],
|
||||
"stream": stream,
|
||||
"max_tokens": 512,
|
||||
}
|
||||
|
||||
response = helper.send_request(api_client, "/v1/chat/completions", request_body)
|
||||
|
||||
# Checkpoint: status code should be 200 or 400
|
||||
assert response.status_code in [
|
||||
200,
|
||||
400,
|
||||
], f"Status code should be 200 or 400, got {response.status_code}"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("stream", [False, True], ids=["non_stream", "stream"])
|
||||
def test_content_array_unicode_text(api_client, request, stream):
|
||||
"""Content array format text field contains Unicode characters, boundary test"""
|
||||
request_body = {
|
||||
"model": "auto",
|
||||
"messages": [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [{"type": "text", "text": "中文🇨🇳日本語🗾العربية🌍"}],
|
||||
}
|
||||
],
|
||||
"stream": stream,
|
||||
"max_tokens": 512,
|
||||
}
|
||||
|
||||
response = helper.send_request(api_client, "/v1/chat/completions", request_body)
|
||||
|
||||
# Checkpoint 1: status code 200
|
||||
assertion.assert_status_code_200(response)
|
||||
|
||||
# Checkpoint 2: streaming response contains [DONE]
|
||||
if stream:
|
||||
assertion.assert_stream_has_done(response.text)
|
||||
@@ -0,0 +1,385 @@
|
||||
import pytest
|
||||
|
||||
from tests.e2e.weekly.single_node.engine_func_test_robot.utility import assertion
|
||||
from tests.e2e.weekly.single_node.engine_func_test_robot.utility import (
|
||||
request_helper as helper,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("stream", [False, True], ids=["non_stream", "stream"])
|
||||
def test_content_simple_string(api_client, request, stream):
|
||||
"""Content is a plain string, request should succeed normally"""
|
||||
request_body = {
|
||||
"model": "auto",
|
||||
"messages": [{"role": "user", "content": "你好，请简单介绍一下自己"}],
|
||||
"stream": stream,
|
||||
"max_tokens": 512,
|
||||
}
|
||||
|
||||
response = helper.send_request(api_client, "/v1/chat/completions", request_body)
|
||||
|
||||
# Checkpoint 1: status code 200
|
||||
assertion.assert_status_code_200(response)
|
||||
|
||||
# Checkpoint 2: streaming response contains [DONE]
|
||||
if stream:
|
||||
assertion.assert_stream_has_done(response.text)
|
||||
|
||||
# Checkpoint 3: finish_reason is valid
|
||||
if stream:
|
||||
finish_reason = assertion.assert_stream_single_finish_reason(response.text)
|
||||
else:
|
||||
finish_reason = response.json()["choices"][0]["finish_reason"]
|
||||
assertion.assert_finish_reason_valid(finish_reason)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("stream", [False, True], ids=["non_stream", "stream"])
|
||||
def test_content_empty_string(api_client, request, stream):
|
||||
"""Content is an empty string, request should succeed normally"""
|
||||
request_body = {
|
||||
"model": "auto",
|
||||
"messages": [{"role": "user", "content": ""}],
|
||||
"stream": stream,
|
||||
"max_tokens": 512,
|
||||
}
|
||||
|
||||
response = helper.send_request(api_client, "/v1/chat/completions", request_body)
|
||||
|
||||
# Checkpoint 1: status code 200
|
||||
assertion.assert_status_code_200(response)
|
||||
|
||||
# Checkpoint 2: streaming response contains [DONE]
|
||||
if stream:
|
||||
assertion.assert_stream_has_done(response.text)
|
||||
|
||||
# Checkpoint 3: finish_reason is stop or length
|
||||
if stream:
|
||||
finish_reason = assertion.assert_stream_single_finish_reason(response.text)
|
||||
else:
|
||||
finish_reason = response.json()["choices"][0]["finish_reason"]
|
||||
assertion.assert_finish_reason_valid(finish_reason)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("stream", [False, True], ids=["non_stream", "stream"])
|
||||
def test_content_null(api_client, request, stream):
|
||||
"""Content is null, request should succeed normally"""
|
||||
request_body = {
|
||||
"model": "auto",
|
||||
"messages": [{"role": "user", "content": None}],
|
||||
"stream": stream,
|
||||
"max_tokens": 512,
|
||||
}
|
||||
|
||||
response = helper.send_request(api_client, "/v1/chat/completions", request_body)
|
||||
|
||||
# Check: error code 400, or finish_reason stop/length both pass
|
||||
if assertion.has_error_code(response):
|
||||
# Error code exists, validate it is 400
|
||||
assertion.assert_error_code_400(response)
|
||||
else:
|
||||
# No error code, check finish_reason is stop or length
|
||||
if stream:
|
||||
assertion.assert_stream_has_done(response.text)
|
||||
|
||||
if stream:
|
||||
finish_reason = assertion.assert_stream_single_finish_reason(response.text)
|
||||
else:
|
||||
finish_reason = response.json()["choices"][0]["finish_reason"]
|
||||
assertion.assert_finish_reason_valid(finish_reason)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("stream", [False, True], ids=["non_stream", "stream"])
|
||||
def test_content_array_empty(api_client, request, stream):
|
||||
"""Content is an empty array [], request should succeed normally"""
|
||||
request_body = {
|
||||
"model": "auto",
|
||||
"messages": [{"role": "user", "content": []}],
|
||||
"stream": stream,
|
||||
"max_tokens": 512,
|
||||
}
|
||||
|
||||
response = helper.send_request(api_client, "/v1/chat/completions", request_body)
|
||||
|
||||
# Checkpoint 1: status code 200
|
||||
assertion.assert_status_code_200(response)
|
||||
|
||||
# Checkpoint 2: streaming response contains [DONE]
|
||||
if stream:
|
||||
assertion.assert_stream_has_done(response.text)
|
||||
|
||||
# Checkpoint 3: finish_reason is stop or length
|
||||
if stream:
|
||||
finish_reason = assertion.assert_stream_single_finish_reason(response.text)
|
||||
else:
|
||||
finish_reason = response.json()["choices"][0]["finish_reason"]
|
||||
assertion.assert_finish_reason_valid(finish_reason)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("stream", [False, True], ids=["non_stream", "stream"])
|
||||
def test_content_missing(api_client, request, stream):
|
||||
"""Message object missing content field, request should succeed normally"""
|
||||
request_body = {
|
||||
"model": "auto",
|
||||
"messages": [
|
||||
{
|
||||
"role": "user"
|
||||
# missing content field
|
||||
}
|
||||
],
|
||||
"stream": stream,
|
||||
"max_tokens": 512,
|
||||
}
|
||||
|
||||
response = helper.send_request(api_client, "/v1/chat/completions", request_body)
|
||||
|
||||
# Checkpoint 1: status code 200
|
||||
assertion.assert_status_code_200(response)
|
||||
|
||||
# Checkpoint 2: streaming response contains [DONE]
|
||||
if stream:
|
||||
assertion.assert_stream_has_done(response.text)
|
||||
|
||||
# Checkpoint 3: finish_reason is stop or length
|
||||
if stream:
|
||||
finish_reason = assertion.assert_stream_single_finish_reason(response.text)
|
||||
else:
|
||||
finish_reason = response.json()["choices"][0]["finish_reason"]
|
||||
assertion.assert_finish_reason_valid(finish_reason)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("stream", [False, True], ids=["non_stream", "stream"])
|
||||
def test_content_with_special_chars(api_client, request, stream):
|
||||
"""Content contains special characters (punctuation, symbols, etc.), request should succeed normally"""
|
||||
request_body = {
|
||||
"model": "auto",
|
||||
"messages": [{"role": "user", "content": "Hello! 你好~ @#$%^&*()_+-=[]{}|;':\",./<>?"}],
|
||||
"stream": stream,
|
||||
"max_tokens": 512,
|
||||
}
|
||||
|
||||
response = helper.send_request(api_client, "/v1/chat/completions", request_body)
|
||||
|
||||
# Checkpoint 1: status code 200
|
||||
assertion.assert_status_code_200(response)
|
||||
|
||||
# Checkpoint 2: streaming response contains [DONE]
|
||||
if stream:
|
||||
assertion.assert_stream_has_done(response.text)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("stream", [False, True], ids=["non_stream", "stream"])
|
||||
def test_content_multiline_text(api_client, request, stream):
|
||||
"""Content contains multiline text (newline characters), request should succeed normally"""
|
||||
request_body = {
|
||||
"model": "auto",
|
||||
"messages": [{"role": "user", "content": "第一行\n第二行\n\n空行后的第三行"}],
|
||||
"stream": stream,
|
||||
"max_tokens": 512,
|
||||
}
|
||||
|
||||
response = helper.send_request(api_client, "/v1/chat/completions", request_body)
|
||||
|
||||
# Checkpoint 1: status code 200
|
||||
assertion.assert_status_code_200(response)
|
||||
|
||||
# Checkpoint 2: streaming response contains [DONE]
|
||||
if stream:
|
||||
assertion.assert_stream_has_done(response.text)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("stream", [False, True], ids=["non_stream", "stream"])
|
||||
def test_content_with_emoji(api_client, request, stream):
|
||||
"""Content contains emoji, request should succeed normally"""
|
||||
request_body = {
|
||||
"model": "auto",
|
||||
"messages": [{"role": "user", "content": "你好👋 很高兴见到你😊 这是一颗星星⭐"}],
|
||||
"stream": stream,
|
||||
"max_tokens": 512,
|
||||
}
|
||||
|
||||
response = helper.send_request(api_client, "/v1/chat/completions", request_body)
|
||||
|
||||
# Checkpoint 1: status code 200
|
||||
assertion.assert_status_code_200(response)
|
||||
|
||||
# Checkpoint 2: streaming response contains [DONE]
|
||||
if stream:
|
||||
assertion.assert_stream_has_done(response.text)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("stream", [False, True], ids=["non_stream", "stream"])
|
||||
def test_content_unicode_chinese(api_client, request, stream):
|
||||
"""Content contains Chinese characters and Unicode characters, request should succeed normally"""
|
||||
request_body = {
|
||||
"model": "auto",
|
||||
"messages": [{"role": "user", "content": " apples 中文测试 日本語テスト 한국어"}],
|
||||
"stream": stream,
|
||||
"max_tokens": 512,
|
||||
}
|
||||
|
||||
response = helper.send_request(api_client, "/v1/chat/completions", request_body)
|
||||
|
||||
# Checkpoint 1: status code 200
|
||||
assertion.assert_status_code_200(response)
|
||||
|
||||
# Checkpoint 2: streaming response contains [DONE]
|
||||
if stream:
|
||||
assertion.assert_stream_has_done(response.text)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("stream", [False, True], ids=["non_stream", "stream"])
|
||||
def test_content_long_text(api_client, request, stream):
|
||||
"""Content is a long text (approx. 1000 characters), request should succeed normally"""
|
||||
long_content = "这是测试文本。" * 100
|
||||
request_body = {
|
||||
"model": "auto",
|
||||
"messages": [{"role": "user", "content": long_content}],
|
||||
"stream": stream,
|
||||
"max_tokens": 512,
|
||||
}
|
||||
|
||||
response = helper.send_request(api_client, "/v1/chat/completions", request_body)
|
||||
|
||||
# Checkpoint 1: status code 200
|
||||
assertion.assert_status_code_200(response)
|
||||
|
||||
# Checkpoint 2: streaming response contains [DONE]
|
||||
if stream:
|
||||
assertion.assert_stream_has_done(response.text)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("stream", [False, True], ids=["non_stream", "stream"])
|
||||
def test_content_code_snippet(api_client, request, stream):
|
||||
"""Content is a code snippet, request should succeed normally"""
|
||||
request_body = {
|
||||
"model": "auto",
|
||||
"messages": [
|
||||
{
|
||||
"role": "user",
|
||||
"content": "```python\ndef hello():\n print('Hello World')\n```请解释这段代码",
|
||||
}
|
||||
],
|
||||
"stream": stream,
|
||||
"max_tokens": 512,
|
||||
}
|
||||
|
||||
response = helper.send_request(api_client, "/v1/chat/completions", request_body)
|
||||
|
||||
# Checkpoint 1: status code 200
|
||||
assertion.assert_status_code_200(response)
|
||||
|
||||
# Checkpoint 2: streaming response contains [DONE]
|
||||
if stream:
|
||||
assertion.assert_stream_has_done(response.text)
|
||||
|
||||
|
||||
# ==================== Content Array Format Tests ====================
|
||||
|
||||
|
||||
@pytest.mark.parametrize("stream", [False, True], ids=["non_stream", "stream"])
|
||||
def test_content_array_text_objects(api_client, request, stream):
|
||||
"""Content is an array of multiple text objects (OpenAI multimodal standard format)"""
|
||||
request_body = {
|
||||
"model": "auto",
|
||||
"messages": [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "text", "text": "你好"},
|
||||
{"type": "text", "text": "你是谁？"},
|
||||
],
|
||||
}
|
||||
],
|
||||
"stream": stream,
|
||||
"max_tokens": 512,
|
||||
}
|
||||
|
||||
response = helper.send_request(api_client, "/v1/chat/completions", request_body)
|
||||
|
||||
# Checkpoint 1: status code 200
|
||||
assertion.assert_status_code_200(response)
|
||||
|
||||
# Checkpoint 2: streaming response contains [DONE]
|
||||
if stream:
|
||||
assertion.assert_stream_has_done(response.text)
|
||||
|
||||
# Checkpoint 3: finish_reason is valid
|
||||
if stream:
|
||||
finish_reason = assertion.assert_stream_single_finish_reason(response.text)
|
||||
else:
|
||||
finish_reason = response.json()["choices"][0]["finish_reason"]
|
||||
assertion.assert_finish_reason_valid(finish_reason)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("stream", [False, True], ids=["non_stream", "stream"])
|
||||
def test_content_array_single_text_object(api_client, request, stream):
|
||||
"""Content is an array with a single text object"""
|
||||
request_body = {
|
||||
"model": "auto",
|
||||
"messages": [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [{"type": "text", "text": "请简单介绍一下自己"}],
|
||||
}
|
||||
],
|
||||
"stream": stream,
|
||||
"max_tokens": 512,
|
||||
}
|
||||
|
||||
response = helper.send_request(api_client, "/v1/chat/completions", request_body)
|
||||
|
||||
# Checkpoint 1: status code 200
|
||||
assertion.assert_status_code_200(response)
|
||||
|
||||
# Checkpoint 2: streaming response contains [DONE]
|
||||
if stream:
|
||||
assertion.assert_stream_has_done(response.text)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("stream", [False, True], ids=["non_stream", "stream"])
|
||||
def test_content_array_empty_text(api_client, request, stream):
|
||||
"""Content is an array format but text is an empty string"""
|
||||
request_body = {
|
||||
"model": "auto",
|
||||
"messages": [{"role": "user", "content": [{"type": "text", "text": ""}]}],
|
||||
"stream": stream,
|
||||
"max_tokens": 512,
|
||||
}
|
||||
|
||||
response = helper.send_request(api_client, "/v1/chat/completions", request_body)
|
||||
|
||||
# Checkpoint: status code should be 200 or 400 (depends on engine implementation)
|
||||
assert response.status_code in [
|
||||
200,
|
||||
400,
|
||||
], f"Status code should be 200 or 400, got {response.status_code}"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("stream", [False, True], ids=["non_stream", "stream"])
|
||||
def test_content_array_many_text_objects(api_client, request, stream):
|
||||
"""Content is an array containing many text objects (boundary test)"""
|
||||
request_body = {
|
||||
"model": "auto",
|
||||
"messages": [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "text", "text": "第一部分内容。"},
|
||||
{"type": "text", "text": "第二部分内容。"},
|
||||
{"type": "text", "text": "第三部分内容。"},
|
||||
{"type": "text", "text": "第四部分内容。"},
|
||||
],
|
||||
}
|
||||
],
|
||||
"stream": stream,
|
||||
"max_tokens": 512,
|
||||
}
|
||||
|
||||
response = helper.send_request(api_client, "/v1/chat/completions", request_body)
|
||||
|
||||
# Checkpoint 1: status code 200
|
||||
assertion.assert_status_code_200(response)
|
||||
|
||||
# Checkpoint 2: streaming response contains [DONE]
|
||||
if stream:
|
||||
assertion.assert_stream_has_done(response.text)
|
||||
@@ -0,0 +1,195 @@
|
||||
import json
|
||||
|
||||
import regex as re
|
||||
|
||||
# think tag definitions
|
||||
THINK_OPEN = "<think>"
|
||||
THINK_CLOSE = "</think>"
|
||||
|
||||
|
||||
class Check:
|
||||
@staticmethod
|
||||
def equal(a, b, msg=""):
|
||||
assert a == b, msg
|
||||
|
||||
@staticmethod
|
||||
def not_equal(a, b, msg=""):
|
||||
assert a != b, msg
|
||||
|
||||
@staticmethod
|
||||
def is_true(v, msg=""):
|
||||
assert v, msg
|
||||
|
||||
@staticmethod
|
||||
def is_in(v, seq, msg=""):
|
||||
assert v in seq, msg
|
||||
|
||||
|
||||
check = Check()
|
||||
|
||||
|
||||
def assert_status_code_200(response, msg=""):
|
||||
"""Verify HTTP status code is 200"""
|
||||
check.equal(response.status_code, 200, f"{msg}Response status code is not 200")
|
||||
|
||||
|
||||
def assert_status_code_400(response, msg=""):
|
||||
"""Verify HTTP status code is 400"""
|
||||
check.equal(response.status_code, 400, f"{msg}Response status code is not 400")
|
||||
|
||||
|
||||
def assert_finish_reason_stop(finish_reason, msg=""):
|
||||
"""Verify finish_reason is stop"""
|
||||
check.equal(finish_reason, "stop", f"{msg}finish_reason is not stop")
|
||||
|
||||
|
||||
def assert_finish_reason_valid(finish_reason, msg=""):
|
||||
"""Verify finish_reason is stop or length"""
|
||||
check.is_in(finish_reason, ["stop", "length"], f"{msg}finish_reason is not stop or length")
|
||||
|
||||
|
||||
def assert_stream_has_done(response_text, msg=""):
|
||||
"""Verify streaming response contains [DONE]"""
|
||||
check.is_true(
|
||||
re.search(r"^data:\s*\[DONE\](?:\n|$)", response_text, re.M),
|
||||
f"{msg}Streaming response does not contain [DONE]",
|
||||
)
|
||||
|
||||
|
||||
def assert_stream_single_finish_reason(response_text, msg=""):
|
||||
"""Verify streaming response has exactly one finish_reason, return its value"""
|
||||
finish_reasons = re.findall(r'finish_reason":\s*"([^"]+)"', response_text, re.M)
|
||||
check.equal(len(finish_reasons), 1, f"{msg}Streaming response has multiple finish_reason values")
|
||||
return finish_reasons[0] if finish_reasons else None
|
||||
|
||||
|
||||
def assert_think_tag_present(response_text, msg=""):
|
||||
"""Verify complete think tag pairs exist"""
|
||||
think_open_count = response_text.count(THINK_OPEN)
|
||||
think_close_count = response_text.count(THINK_CLOSE)
|
||||
check.equal(
|
||||
think_open_count,
|
||||
think_close_count,
|
||||
f"{msg}think tags are not balanced, OPEN: {think_open_count}, CLOSE: {think_close_count}",
|
||||
)
|
||||
check.equal(think_open_count, 1, f"{msg}No think tag present")
|
||||
|
||||
|
||||
def assert_no_think_tag(response_text, msg=""):
|
||||
"""Verify think tags do not exist"""
|
||||
check.equal(response_text.count(THINK_OPEN), 0, f"{msg}think tag exists")
|
||||
|
||||
|
||||
def assert_json_response_content(response_text, msg=""):
|
||||
"""Verify response content is valid JSON (after filtering think tags)"""
|
||||
pattern = rf"\s*{re.escape(THINK_OPEN)}[\s\S]*?{re.escape(THINK_CLOSE)}"
|
||||
json_str = re.sub(pattern, "", response_text)
|
||||
match = re.search(r"(\{.*\})\s*(?:$|`|```)$", json_str, re.S)
|
||||
check.is_true(match, f"{msg}Content is not in JSON format")
|
||||
if match:
|
||||
json.loads(match.group(1))
|
||||
|
||||
|
||||
def has_error_code(response):
|
||||
"""Determine if the response contains an error code"""
|
||||
content_type = response.headers.get("Content-Type", "")
|
||||
if "application/json" in content_type:
|
||||
response_json = response.json()
|
||||
error_code = response_json.get("error", {}).get("code") or response_json.get("code")
|
||||
return error_code is not None
|
||||
elif "text/event-stream" in content_type or "text/plain" in content_type:
|
||||
match = re.search(r"\"code\"\s?:\s?(\d+)", response.text, re.M)
|
||||
return match is not None
|
||||
return False
|
||||
|
||||
|
||||
def assert_error_code_400(response, msg=""):
|
||||
"""Verify error code is 400"""
|
||||
if "application/json" in response.headers.get("Content-Type", ""):
|
||||
error_code = response.json().get("error", {}).get("code") or response.json().get("code")
|
||||
check.equal(error_code, 400, f"{msg}Error code is not 400")
|
||||
elif "text/event-stream" in response.headers.get("Content-Type", ""):
|
||||
match = re.search(r"\"code\"\s?:\s?(\d+)", response.text, re.M)
|
||||
if match:
|
||||
check.equal(int(match.group(1)), 400, f"{msg}Streaming response error code is not 400")
|
||||
|
||||
|
||||
def assert_error_code_422(response, msg=""):
|
||||
"""Verify error code is 422 Unprocessable Entity (data validation failure)"""
|
||||
if "application/json" in response.headers.get("Content-Type", ""):
|
||||
error_code = response.json().get("error", {}).get("code") or response.json().get("code")
|
||||
check.equal(error_code, 422, f"{msg}Error code is not 422 (Unprocessable Entity - data validation failure)")
|
||||
elif "text/event-stream" in response.headers.get("Content-Type", ""):
|
||||
match = re.search(r"\"code\"\s?:\s?(\d+)", response.text, re.M)
|
||||
if match:
|
||||
check.equal(int(match.group(1)), 422, f"{msg}Streaming response error code is not 422")
|
||||
|
||||
|
||||
def assert_error_code_not_500(response, msg=""):
|
||||
"""If response body contains an error code, verify it is not 500"""
|
||||
content_type = response.headers.get("Content-Type", "")
|
||||
if "application/json" in content_type:
|
||||
error_code = response.json().get("error", {}).get("code") or response.json().get("code")
|
||||
if error_code is not None:
|
||||
check.not_equal(error_code, 500, f"{msg}Error code should not be 500, actual: {error_code}")
|
||||
elif "text/event-stream" in content_type:
|
||||
match = re.search(r"\"code\"\s?:\s?(\d+)", response.text, re.M)
|
||||
if match:
|
||||
error_code = int(match.group(1))
|
||||
check.not_equal(error_code, 500, f"{msg}Error code should not be 500, actual: {error_code}")
|
||||
|
||||
|
||||
def assert_image_edit_response_fields(response, msg=""):
|
||||
"""Verify completeness of response fields for image edit API
|
||||
|
||||
Args:
|
||||
response: HTTP response object
|
||||
msg: Prefix for error messages
|
||||
"""
|
||||
resp_json = response.json()
|
||||
|
||||
# Verify top-level fields
|
||||
check.is_true("created" in resp_json, f"{msg}Response should contain created field")
|
||||
check.is_true("data" in resp_json, f"{msg}Response should contain data field")
|
||||
check.is_true("output_format" in resp_json, f"{msg}Response should contain output_format field")
|
||||
check.is_true("size" in resp_json, f"{msg}Response should contain size field")
|
||||
|
||||
# Verify data array
|
||||
data = resp_json.get("data", [])
|
||||
check.is_true(len(data) > 0, f"{msg}data should contain at least one result")
|
||||
|
||||
# Verify fields of each data array element
|
||||
for idx, item in enumerate(data):
|
||||
has_b64 = "b64_json" in item and item["b64_json"]
|
||||
has_url = "url" in item and item["url"]
|
||||
check.is_true(has_b64 or has_url, f"{msg}data[{idx}] should contain b64_json or url field")
|
||||
check.is_true("revised_prompt" in item, f"{msg}data[{idx}] should contain revised_prompt field")
|
||||
|
||||
return resp_json
|
||||
|
||||
|
||||
def assert_top_logprobs_count(response, top_logprobs_value, msg=""):
|
||||
"""Verify the number of top_logprobs in logprobs"""
|
||||
content_type = response.headers.get("Content-Type", "")
|
||||
|
||||
if "application/json" in content_type:
|
||||
logprobs_content_list = response.json()["choices"][0]["logprobs"]["content"]
|
||||
for item_dict in logprobs_content_list:
|
||||
check.equal(
|
||||
len(item_dict.get("top_logprobs")),
|
||||
top_logprobs_value,
|
||||
f"{msg}logprobs top_logprobs length is not {top_logprobs_value}",
|
||||
)
|
||||
elif "text/event-stream" in content_type:
|
||||
chunk_list = re.findall(r"^data:\s*(.*)(?:\n|$)", response.text, re.M)[1:-1]
|
||||
for chunk_item in chunk_list:
|
||||
chunk_json = json.loads(chunk_item)
|
||||
content = chunk_json["choices"][0]["delta"].get("content", "")
|
||||
if content:
|
||||
logprobs_content_list = chunk_json["choices"][0]["logprobs"]["content"]
|
||||
for item_dict in logprobs_content_list:
|
||||
check.equal(
|
||||
len(item_dict.get("top_logprobs")),
|
||||
top_logprobs_value,
|
||||
f"{msg}Streaming logprobs top_logprobs length is not {top_logprobs_value}",
|
||||
)
|
||||
@@ -0,0 +1,25 @@
|
||||
import requests
|
||||
from requests.exceptions import RequestException
|
||||
|
||||
|
||||
class HTTPClient:
|
||||
def __init__(self, base_url=None, timeout=36000):
|
||||
self.base_url = base_url.rstrip("/") if base_url else ""
|
||||
self.timeout = timeout
|
||||
|
||||
def get(self, endpoint, params=None, headers=None):
|
||||
url = f"{self.base_url}/{endpoint.lstrip('/')}"
|
||||
try:
|
||||
response = requests.get(url, params=params, headers=headers, timeout=self.timeout)
|
||||
response.raise_for_status()
|
||||
return response
|
||||
except RequestException as e:
|
||||
raise AssertionError(f"GET {url} failed: {str(e)}")
|
||||
|
||||
def post(self, endpoint, json=None, data=None, files=None, headers=None):
|
||||
url = f"{self.base_url}/{endpoint.lstrip('/')}"
|
||||
try:
|
||||
response = requests.post(url, json=json, data=data, files=files, headers=headers, timeout=self.timeout)
|
||||
return response
|
||||
except RequestException as e:
|
||||
raise AssertionError(f"POST {url} failed: {str(e)}")
|
||||
@@ -0,0 +1,3 @@
|
||||
def send_request(api_client, uri, request_body):
|
||||
"""Send request and return response object"""
|
||||
return api_client.post(uri, json=request_body, headers={"Content-Type": "application/json"})
|
||||
Reference in New Issue
Block a user