dylanyunlong/project_6
1
0
Fork 0
Code Issues Pull Requests Actions Projects Releases Wiki Activity

ref(upstream): FULL TREE — Deep-Spark xllm (1470) + ds_vllm csrc/models (703)

Browse Source 
Replaces cherry-picked upstream_ref with complete source trees.

xllm/ — Iluvatar official C++ inference engine (15MB, 1470 files)
  Complete: kernels → layers → models → runtime → scheduler → api
  Excluded: .git, binary images, third_party submodule checkouts

ds_vllm/ — Iluvatar official vllm fork (8MB, 703 files)
  Included: csrc/ (ALL CUDA kernels), fused_moe/, qwen3_5 model, _custom_ops
  Excluded: tests, benchmarks, docs, examples (not needed for reference)

Critical call chains now fully traceable:
  MoE: moe_topk_softmax_kernels.cuh → ixformer.h → fused_moe.cpp → layer
  GDN: qwen3_gated_delta_net_base.cpp → qwen3_5_gated_delta_net.cpp
  Attention: ixformer.h → xllm_paged_attention → attention.cpp
This commit is contained in:
EX Engine
2026-08-10 02:53:54 +00:00
parent 9e4fb3712f
commit 002f9879b2
2179 changed files with 494021 additions and 79 deletions
Download Patch File Download Diff File Expand all files Collapse all files

68
upstream_ref/xllm/tests/api_service/CMakeLists.txt Normal file
View File

@@ -0,0 +1,68 @@
include(cc_test)
cc_test(
NAME
anthropic_protocol_test
SRCS
anthropic_protocol_test.cpp
DEPS
proto::xllm_proto
GTest::gtest
GTest::gtest_main
nlohmann_json::nlohmann_json
glog::glog
)
# Integration test for Anthropic service - requires running server.
# Tests are prefixed with DISABLED_ in code to skip by default.
# Run manually: ./anthropic_service_test --gtest_also_run_disabled_tests
cc_test(
NAME
anthropic_service_test
SRCS
anthropic_service_test.cpp
DEPS
GTest::gtest
GTest::gtest_main
nlohmann_json::nlohmann_json
glog::glog
)
target_link_libraries(anthropic_service_test PRIVATE brpc leveldb::leveldb OpenSSL::SSL OpenSSL::Crypto protobuf::libprotobuf)
add_dependencies(anthropic_service_test brpc-static)
cc_test(
NAME
api_service_test
SRCS
chat_json_parser_test.cpp
DEPS
api_service
GTest::gtest_main
nlohmann_json::nlohmann_json
)
cc_test(
NAME
sample_service_impl_test
SRCS
sample_service_impl_test.cpp
DEPS
api_service
GTest::gtest_main
)
# Integration smoke tests for OpenAI-compatible endpoints. These require a
# running server and are disabled by default in code.
cc_test(
NAME
openai_service_test
SRCS
openai_service_test.cpp
DEPS
GTest::gtest
GTest::gtest_main
nlohmann_json::nlohmann_json
glog::glog
)
target_link_libraries(openai_service_test PRIVATE brpc leveldb::leveldb OpenSSL::SSL OpenSSL::Crypto protobuf::libprotobuf)
add_dependencies(openai_service_test brpc-static)

600
upstream_ref/xllm/tests/api_service/anthropic_protocol_test.cpp Normal file
View File

@@ -0,0 +1,600 @@
/* Copyright 2026 The xLLM Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
https://github.com/jd-opensource/xllm/blob/main/LICENSE
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#include <glog/logging.h>
#include <google/protobuf/util/json_util.h>
#include <gtest/gtest.h>
#include <nlohmann/json.hpp>
#include <string>
#include <vector>
#include "anthropic.pb.h"
namespace xllm {
namespace {
using proto::AnthropicContentBlock;
using proto::AnthropicContentBlockList;
using proto::AnthropicMessage;
using proto::AnthropicMessagesRequest;
using proto::AnthropicMessagesResponse;
using proto::AnthropicTool;
using proto::AnthropicToolChoice;
using proto::AnthropicUsage;
class AnthropicProtocolTest : public ::testing::Test {
protected:
void SetUp() override {}
// Helper to convert nlohmann::json to google::protobuf::Struct
static google::protobuf::Struct json_to_struct(const nlohmann::json& j) {
google::protobuf::Struct pb_struct;
std::string json_str = j.dump();
google::protobuf::util::JsonStringToMessage(json_str, &pb_struct);
return pb_struct;
}
// Helper to convert google::protobuf::Struct to nlohmann::json
static nlohmann::json struct_to_json(
const google::protobuf::Struct& pb_struct) {
std::string json_str;
google::protobuf::util::JsonPrintOptions options;
options.preserve_proto_field_names = true;
google::protobuf::util::MessageToJsonString(pb_struct, &json_str, options);
return nlohmann::json::parse(json_str);
}
};
// Test AnthropicContentBlock with text content
TEST_F(AnthropicProtocolTest, AnthropicContentBlockText) {
AnthropicContentBlock content_block;
content_block.set_type("text");
content_block.set_text("Hello, world!");
EXPECT_EQ(content_block.type(), "text");
EXPECT_EQ(content_block.text(), "Hello, world!");
EXPECT_FALSE(content_block.has_source());
EXPECT_FALSE(content_block.has_id());
EXPECT_FALSE(content_block.has_name());
EXPECT_FALSE(content_block.has_input());
EXPECT_EQ(content_block.tool_result_content_case(),
AnthropicContentBlock::TOOL_RESULT_CONTENT_NOT_SET);
EXPECT_FALSE(content_block.has_is_error());
}
// Test AnthropicContentBlock with tool use content
TEST_F(AnthropicProtocolTest, AnthropicContentBlockToolUse) {
AnthropicContentBlock content_block;
content_block.set_type("tool_use");
content_block.set_id("toolu_123");
content_block.set_name("get_weather");
nlohmann::json input_json = {{"location", "Paris"}, {"unit", "celsius"}};
*content_block.mutable_input() = json_to_struct(input_json);
EXPECT_EQ(content_block.type(), "tool_use");
EXPECT_EQ(content_block.id(), "toolu_123");
EXPECT_EQ(content_block.name(), "get_weather");
// Verify input
auto parsed_input = struct_to_json(content_block.input());
EXPECT_EQ(parsed_input["location"], "Paris");
EXPECT_EQ(parsed_input["unit"], "celsius");
}
// Test AnthropicContentBlock with tool result content
TEST_F(AnthropicProtocolTest, AnthropicContentBlockToolResult) {
AnthropicContentBlock content_block;
content_block.set_type("tool_result");
content_block.set_id("call_123");
content_block.set_content_string("The weather in Paris is sunny.");
content_block.set_is_error(false);
EXPECT_EQ(content_block.type(), "tool_result");
EXPECT_EQ(content_block.id(), "call_123");
EXPECT_EQ(content_block.content_string(), "The weather in Paris is sunny.");
EXPECT_FALSE(content_block.is_error());
}
// Test AnthropicMessage for user role
TEST_F(AnthropicProtocolTest, AnthropicMessageUser) {
AnthropicMessage message;
message.set_role("user");
message.set_content_string("Hello, assistant!");
EXPECT_EQ(message.role(), "user");
EXPECT_EQ(message.content_string(), "Hello, assistant!");
}
// Test AnthropicMessage for assistant role with text content
TEST_F(AnthropicProtocolTest, AnthropicMessageAssistantText) {
AnthropicMessage message;
message.set_role("assistant");
// Create content block list
auto* content_blocks = message.mutable_content_blocks();
auto* text_block = content_blocks->add_blocks();
text_block->set_type("text");
text_block->set_text("Hello, user!");
EXPECT_EQ(message.role(), "assistant");
EXPECT_TRUE(message.has_content_blocks());
EXPECT_EQ(message.content_blocks().blocks_size(), 1);
EXPECT_EQ(message.content_blocks().blocks(0).type(), "text");
EXPECT_EQ(message.content_blocks().blocks(0).text(), "Hello, user!");
}
// Test AnthropicTool model
TEST_F(AnthropicProtocolTest, AnthropicTool) {
nlohmann::json tool_schema = {
{"type", "object"},
{"properties",
{{"location", {{"type", "string"}}},
{"unit", {{"type", "string"}, {"enum", {"celsius", "fahrenheit"}}}}}},
{"required", {"location"}}};
AnthropicTool tool;
tool.set_name("get_weather");
tool.set_description("Get the current weather for a location");
*tool.mutable_input_schema() = json_to_struct(tool_schema);
EXPECT_EQ(tool.name(), "get_weather");
EXPECT_EQ(tool.description(), "Get the current weather for a location");
// Verify input_schema
auto parsed_schema = struct_to_json(tool.input_schema());
EXPECT_EQ(parsed_schema["type"], "object");
EXPECT_TRUE(parsed_schema.contains("properties"));
EXPECT_TRUE(parsed_schema["properties"].contains("location"));
}
// Test AnthropicToolChoice with auto type
TEST_F(AnthropicProtocolTest, AnthropicToolChoiceAuto) {
AnthropicToolChoice tool_choice;
tool_choice.set_type("auto");
EXPECT_EQ(tool_choice.type(), "auto");
EXPECT_FALSE(tool_choice.has_name());
}
// Test AnthropicToolChoice with specific tool
TEST_F(AnthropicProtocolTest, AnthropicToolChoiceSpecific) {
AnthropicToolChoice tool_choice;
tool_choice.set_type("tool");
tool_choice.set_name("get_weather");
EXPECT_EQ(tool_choice.type(), "tool");
EXPECT_EQ(tool_choice.name(), "get_weather");
}
// Test basic AnthropicMessagesRequest
TEST_F(AnthropicProtocolTest, AnthropicMessagesRequestBasic) {
AnthropicMessagesRequest request;
request.set_model("my_model");
request.set_max_tokens(100);
// Add message
auto* message = request.add_messages();
message->set_role("user");
message->set_content_string("What is the weather like today?");
EXPECT_EQ(request.model(), "my_model");
EXPECT_EQ(request.messages_size(), 1);
EXPECT_EQ(request.messages(0).role(), "user");
EXPECT_EQ(request.max_tokens(), 100);
EXPECT_FALSE(request.has_stream());
EXPECT_FALSE(request.has_system_string());
EXPECT_FALSE(request.has_temperature());
}
// Test AnthropicMessagesRequest with system prompt
TEST_F(AnthropicProtocolTest, AnthropicMessagesRequestWithSystem) {
AnthropicMessagesRequest request;
request.set_model("my_model");
request.set_max_tokens(100);
request.set_system_string("You are a helpful weather assistant.");
// Add message
auto* message = request.add_messages();
message->set_role("user");
message->set_content_string("What is the weather like today?");
EXPECT_EQ(request.system_string(), "You are a helpful weather assistant.");
}
// Test AnthropicMessagesRequest with tools
TEST_F(AnthropicProtocolTest, AnthropicMessagesRequestWithTools) {
AnthropicMessagesRequest request;
request.set_model("my_model");
request.set_max_tokens(100);
// Add message
auto* message = request.add_messages();
message->set_role("user");
message->set_content_string("What is the weather in Paris?");
// Add tool
nlohmann::json tool_schema = {
{"type", "object"},
{"properties",
{{"location", {{"type", "string"}}},
{"unit", {{"type", "string"}, {"enum", {"celsius", "fahrenheit"}}}}}},
{"required", {"location"}}};
auto* tool = request.add_tools();
tool->set_name("get_weather");
tool->set_description("Get the current weather for a location");
*tool->mutable_input_schema() = json_to_struct(tool_schema);
// Set tool choice
auto* tool_choice = request.mutable_tool_choice();
tool_choice->set_type("auto");
EXPECT_EQ(request.tools_size(), 1);
EXPECT_EQ(request.tools(0).name(), "get_weather");
EXPECT_EQ(request.tool_choice().type(), "auto");
}
// Test AnthropicMessagesRequest with streaming
TEST_F(AnthropicProtocolTest, AnthropicMessagesRequestStreaming) {
AnthropicMessagesRequest request;
request.set_model("my_model");
request.set_max_tokens(1000);
request.set_stream(true);
// Add message
auto* message = request.add_messages();
message->set_role("user");
message->set_content_string("Tell me a story.");
EXPECT_TRUE(request.stream());
}
// Test AnthropicMessagesRequest with all parameters
TEST_F(AnthropicProtocolTest, AnthropicMessagesRequestAllParams) {
AnthropicMessagesRequest request;
request.set_model("my_model");
request.set_max_tokens(100);
request.set_temperature(0.7f);
request.set_top_p(0.9f);
request.set_top_k(50);
// Add message
auto* message = request.add_messages();
message->set_role("user");
message->set_content_string("Hello");
EXPECT_FLOAT_EQ(request.temperature(), 0.7f);
EXPECT_FLOAT_EQ(request.top_p(), 0.9f);
EXPECT_EQ(request.top_k(), 50);
}
// Test AnthropicUsage model
TEST_F(AnthropicProtocolTest, AnthropicUsage) {
AnthropicUsage usage;
usage.set_input_tokens(50);
usage.set_output_tokens(75);
EXPECT_EQ(usage.input_tokens(), 50);
EXPECT_EQ(usage.output_tokens(), 75);
EXPECT_FALSE(usage.has_cache_creation_input_tokens());
EXPECT_FALSE(usage.has_cache_read_input_tokens());
}
// Test AnthropicUsage model with cache tokens
TEST_F(AnthropicProtocolTest, AnthropicUsageWithCache) {
AnthropicUsage usage;
usage.set_input_tokens(50);
usage.set_output_tokens(75);
usage.set_cache_creation_input_tokens(25);
usage.set_cache_read_input_tokens(15);
EXPECT_EQ(usage.input_tokens(), 50);
EXPECT_EQ(usage.output_tokens(), 75);
EXPECT_EQ(usage.cache_creation_input_tokens(), 25);
EXPECT_EQ(usage.cache_read_input_tokens(), 15);
}
// Test AnthropicMessagesResponse model
TEST_F(AnthropicProtocolTest, AnthropicMessagesResponse) {
AnthropicMessagesResponse response;
response.set_id("msg_123");
response.set_type("message");
response.set_role("assistant");
response.set_model("my_model");
response.set_stop_reason("end_turn");
// Add content block
auto* content_block = response.add_content();
content_block->set_type("text");
content_block->set_text("Hello, user!");
// Set usage
auto* usage = response.mutable_usage();
usage->set_input_tokens(10);
usage->set_output_tokens(20);
EXPECT_EQ(response.id(), "msg_123");
EXPECT_EQ(response.type(), "message");
EXPECT_EQ(response.role(), "assistant");
EXPECT_EQ(response.content_size(), 1);
EXPECT_EQ(response.content(0).type(), "text");
EXPECT_EQ(response.content(0).text(), "Hello, user!");
EXPECT_EQ(response.model(), "my_model");
EXPECT_EQ(response.stop_reason(), "end_turn");
EXPECT_EQ(response.usage().input_tokens(), 10);
EXPECT_EQ(response.usage().output_tokens(), 20);
}
// Test AnthropicMessagesResponse with specific ID
TEST_F(AnthropicProtocolTest, AnthropicMessagesResponseAutoId) {
AnthropicMessagesResponse response;
response.set_id("msg_test123");
response.set_type("message");
response.set_role("assistant");
response.set_model("my_model");
// Add content block
auto* content_block = response.add_content();
content_block->set_type("text");
content_block->set_text("Hello!");
// Set usage
auto* usage = response.mutable_usage();
usage->set_input_tokens(5);
usage->set_output_tokens(10);
EXPECT_EQ(response.id(), "msg_test123");
// Check that ID starts with "msg_"
EXPECT_EQ(response.id().substr(0, 4), "msg_");
}
// Test AnthropicTool with default input_schema type
TEST_F(AnthropicProtocolTest, AnthropicToolInputSchemaDefault) {
nlohmann::json tool_schema = {
{"properties", {{"param", {{"type", "string"}}}}}};
AnthropicTool tool;
tool.set_name("simple_tool");
*tool.mutable_input_schema() = json_to_struct(tool_schema);
auto parsed_schema = struct_to_json(tool.input_schema());
EXPECT_TRUE(parsed_schema.contains("properties"));
}
// Test message with multiple content blocks
TEST_F(AnthropicProtocolTest, AnthropicMessageMultipleContentBlocks) {
AnthropicMessage message;
message.set_role("assistant");
auto* content_blocks = message.mutable_content_blocks();
// Add text block
auto* text_block = content_blocks->add_blocks();
text_block->set_type("text");
text_block->set_text("I'll help you check the weather.");
// Add tool use block
auto* tool_block = content_blocks->add_blocks();
tool_block->set_type("tool_use");
tool_block->set_id("toolu_456");
tool_block->set_name("get_weather");
nlohmann::json input_json = {{"location", "Tokyo"}};
*tool_block->mutable_input() = json_to_struct(input_json);
EXPECT_EQ(message.content_blocks().blocks_size(), 2);
EXPECT_EQ(message.content_blocks().blocks(0).type(), "text");
EXPECT_EQ(message.content_blocks().blocks(1).type(), "tool_use");
EXPECT_EQ(message.content_blocks().blocks(1).id(), "toolu_456");
}
// Test AnthropicMessagesRequest with stop_sequences
TEST_F(AnthropicProtocolTest, AnthropicMessagesRequestStopSequences) {
AnthropicMessagesRequest request;
request.set_model("my_model");
request.set_max_tokens(100);
// Add stop sequences
request.add_stop_sequences("END");
request.add_stop_sequences("STOP");
// Add message
auto* message = request.add_messages();
message->set_role("user");
message->set_content_string("Generate text until END");
EXPECT_EQ(request.stop_sequences_size(), 2);
EXPECT_EQ(request.stop_sequences(0), "END");
EXPECT_EQ(request.stop_sequences(1), "STOP");
}
// Test JSON serialization round-trip
TEST_F(AnthropicProtocolTest, JsonSerializationRoundTrip) {
AnthropicMessagesRequest original;
original.set_model("my_model");
original.set_max_tokens(100);
original.set_temperature(0.7f);
auto* message = original.add_messages();
message->set_role("user");
message->set_content_string("Hello!");
// Serialize to JSON
std::string json_str;
google::protobuf::util::JsonPrintOptions options;
options.preserve_proto_field_names = true;
auto status =
google::protobuf::util::MessageToJsonString(original, &json_str, options);
ASSERT_TRUE(status.ok());
// Deserialize back
AnthropicMessagesRequest parsed;
status = google::protobuf::util::JsonStringToMessage(json_str, &parsed);
ASSERT_TRUE(status.ok());
// Verify
EXPECT_EQ(parsed.model(), original.model());
EXPECT_EQ(parsed.max_tokens(), original.max_tokens());
EXPECT_FLOAT_EQ(parsed.temperature(), original.temperature());
EXPECT_EQ(parsed.messages_size(), 1);
EXPECT_EQ(parsed.messages(0).content_string(), "Hello!");
}
// Test AnthropicContentBlock with image source
TEST_F(AnthropicProtocolTest, AnthropicContentBlockImage) {
AnthropicContentBlock content_block;
content_block.set_type("image");
nlohmann::json source_json = {
{"type", "base64"}, {"media_type", "image/png"}, {"data", "xxxxxx"}};
*content_block.mutable_source() = json_to_struct(source_json);
EXPECT_EQ(content_block.type(), "image");
EXPECT_TRUE(content_block.has_source());
auto parsed_source = struct_to_json(content_block.source());
EXPECT_EQ(parsed_source["type"], "base64");
EXPECT_EQ(parsed_source["media_type"], "image/png");
}
// Test tool result with list content
TEST_F(AnthropicProtocolTest, AnthropicContentBlockToolResultList) {
AnthropicContentBlock content_block;
content_block.set_type("tool_result");
content_block.set_id("call_789");
// Use content_list for complex content
auto* content_list = content_block.mutable_content_list();
nlohmann::json item_json = {{"type", "text"}, {"text", "Result item 1"}};
*content_list->add_items() = json_to_struct(item_json);
EXPECT_EQ(content_block.type(), "tool_result");
EXPECT_TRUE(content_block.has_content_list());
EXPECT_EQ(content_block.content_list().items_size(), 1);
}
// Test response with tool_use stop_reason
TEST_F(AnthropicProtocolTest, AnthropicMessagesResponseToolUse) {
AnthropicMessagesResponse response;
response.set_id("msg_tool_123");
response.set_type("message");
response.set_role("assistant");
response.set_model("my_model");
response.set_stop_reason("tool_use");
// Add text content
auto* text_block = response.add_content();
text_block->set_type("text");
text_block->set_text("Let me check the weather for you.");
// Add tool_use content
auto* tool_block = response.add_content();
tool_block->set_type("tool_use");
tool_block->set_id("toolu_weather_1");
tool_block->set_name("get_weather");
nlohmann::json input_json = {{"location", "San Francisco"},
{"unit", "fahrenheit"}};
*tool_block->mutable_input() = json_to_struct(input_json);
// Set usage
auto* usage = response.mutable_usage();
usage->set_input_tokens(30);
usage->set_output_tokens(45);
EXPECT_EQ(response.stop_reason(), "tool_use");
EXPECT_EQ(response.content_size(), 2);
EXPECT_EQ(response.content(0).type(), "text");
EXPECT_EQ(response.content(1).type(), "tool_use");
EXPECT_EQ(response.content(1).name(), "get_weather");
auto parsed_input = struct_to_json(response.content(1).input());
EXPECT_EQ(parsed_input["location"], "San Francisco");
}
// Test request with system blocks
TEST_F(AnthropicProtocolTest, AnthropicMessagesRequestSystemBlocks) {
AnthropicMessagesRequest request;
request.set_model("my_model");
request.set_max_tokens(100);
// Use system_blocks instead of system_string
auto* system_blocks = request.mutable_system_blocks();
auto* block = system_blocks->add_blocks();
block->set_type("text");
block->set_text("You are a helpful assistant specialized in weather.");
auto* message = request.add_messages();
message->set_role("user");
message->set_content_string("What's the weather?");
EXPECT_TRUE(request.has_system_blocks());
EXPECT_FALSE(request.has_system_string());
EXPECT_EQ(request.system_blocks().blocks_size(), 1);
EXPECT_EQ(request.system_blocks().blocks(0).text(),
"You are a helpful assistant specialized in weather.");
}
// Test AnthropicToolChoice with "any" type
TEST_F(AnthropicProtocolTest, AnthropicToolChoiceAny) {
AnthropicToolChoice tool_choice;
tool_choice.set_type("any");
EXPECT_EQ(tool_choice.type(), "any");
EXPECT_FALSE(tool_choice.has_name());
}
// Test complex conversation with multiple turns
TEST_F(AnthropicProtocolTest, AnthropicMessagesRequestMultiTurn) {
AnthropicMessagesRequest request;
request.set_model("my_model");
request.set_max_tokens(200);
// User message 1
auto* msg1 = request.add_messages();
msg1->set_role("user");
msg1->set_content_string("What's the weather in Paris?");
// Assistant message with tool use
auto* msg2 = request.add_messages();
msg2->set_role("assistant");
auto* content_blocks2 = msg2->mutable_content_blocks();
auto* tool_use_block = content_blocks2->add_blocks();
tool_use_block->set_type("tool_use");
tool_use_block->set_id("toolu_123");
tool_use_block->set_name("get_weather");
nlohmann::json input_json = {{"location", "Paris"}};
*tool_use_block->mutable_input() = json_to_struct(input_json);
// User message with tool result
auto* msg3 = request.add_messages();
msg3->set_role("user");
auto* content_blocks3 = msg3->mutable_content_blocks();
auto* tool_result_block = content_blocks3->add_blocks();
tool_result_block->set_type("tool_result");
tool_result_block->set_id("toolu_123");
tool_result_block->set_content_string("The weather in Paris is 22°C, sunny.");
EXPECT_EQ(request.messages_size(), 3);
EXPECT_EQ(request.messages(0).role(), "user");
EXPECT_EQ(request.messages(1).role(), "assistant");
EXPECT_EQ(request.messages(2).role(), "user");
EXPECT_EQ(request.messages(1).content_blocks().blocks(0).type(), "tool_use");
EXPECT_EQ(request.messages(2).content_blocks().blocks(0).type(),
"tool_result");
}
} // namespace
} // namespace xllm

1329
upstream_ref/xllm/tests/api_service/anthropic_service_test.cpp Normal file
View File

File diff suppressed because it is too large Load Diff

383
upstream_ref/xllm/tests/api_service/chat_json_parser_test.cpp Normal file
View File

@@ -0,0 +1,383 @@
/* Copyright 2026 The xLLM Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
https://github.com/jd-opensource/xllm/blob/main/LICENSE
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#include "api_service/chat_json_parser.h"
#include <gtest/gtest.h>
#include <nlohmann/json.hpp>
namespace xllm {
class PreprocessChatJsonTest : public ::testing::Test {
protected:
void expect_success(const std::string& input,
const ChatJsonParser& parser,
const std::string& expected_output) {
auto [status, result] = parser.preprocess(input);
ASSERT_TRUE(status.ok()) << "Unexpected error: " << status.message();
auto result_json = nlohmann::json::parse(result);
auto expected_json = nlohmann::json::parse(expected_output);
EXPECT_EQ(result_json, expected_json);
}
void expect_error(const std::string& input,
const ChatJsonParser& parser,
const std::string& expected_error_substring) {
auto [status, result] = parser.preprocess(input);
ASSERT_FALSE(status.ok()) << "Expected error but got success";
EXPECT_NE(status.message().find(expected_error_substring),
std::string::npos)
<< "Error message '" << status.message()
<< "' does not contain expected substring '" << expected_error_substring
<< "'";
}
};
// =============================================================================
// Basic functionality tests
// =============================================================================
TEST_F(PreprocessChatJsonTest, PassThroughNonArrayContent) {
// String content should pass through unchanged
std::string input = R"({
"messages": [{"role": "user", "content": "Hello"}]
})";
LlmChatJsonParser llm_parser;
VlmChatJsonParser vlm_parser;
expect_success(input, llm_parser, input);
expect_success(input, vlm_parser, input);
}
TEST_F(PreprocessChatJsonTest, PassThroughNoMessages) {
// JSON without messages field should pass through
std::string input = R"({"model": "test"})";
LlmChatJsonParser llm_parser;
expect_success(input, llm_parser, input);
}
TEST_F(PreprocessChatJsonTest, CombineTextArrayIntoString) {
// Array of text items should be combined into single string for
// non-multimodal
std::string input = R"({
"messages": [{
"role": "user",
"content": [
{"type": "text", "text": "Hello"},
{"type": "text", "text": "World"}
]
}]
})";
std::string expected = R"({
"messages": [{"role": "user", "content": "Hello\nWorld"}]
})";
LlmChatJsonParser llm_parser;
VlmChatJsonParser vlm_parser;
expect_success(input, llm_parser, expected);
// For multimodal, array is preserved (not combined)
expect_success(input, vlm_parser, input);
}
TEST_F(PreprocessChatJsonTest, SingleTextItemCombined) {
// Single text item in array should be converted to string for non-multimodal
std::string input = R"({
"messages": [{
"role": "user",
"content": [{"type": "text", "text": "Hello"}]
}]
})";
std::string expected = R"({
"messages": [{"role": "user", "content": "Hello"}]
})";
LlmChatJsonParser llm_parser;
VlmChatJsonParser vlm_parser;
expect_success(input, llm_parser, expected);
// For multimodal, array is preserved
expect_success(input, vlm_parser, input);
}
// =============================================================================
// Multimodal content tests (Issue #801)
// =============================================================================
TEST_F(PreprocessChatJsonTest, ImageUrlPassesThroughOnMultimodal) {
// image_url content should pass through unchanged on multimodal endpoint
std::string input = R"({
"messages": [{
"role": "user",
"content": [
{"type": "image_url", "image_url": {"url": "data:image/jpeg;base64,abc"}},
{"type": "text", "text": "What is this?"}
]
}]
})";
VlmChatJsonParser vlm_parser;
// Should pass through unchanged for multimodal
expect_success(input, vlm_parser, input);
}
TEST_F(PreprocessChatJsonTest, ImageUrlErrorsOnTextOnly) {
// image_url content should error on text-only endpoint with helpful message
std::string input = R"({
"messages": [{
"role": "user",
"content": [
{"type": "image_url", "image_url": {"url": "data:image/jpeg;base64,abc"}},
{"type": "text", "text": "What is this?"}
]
}]
})";
LlmChatJsonParser llm_parser;
expect_error(input, llm_parser, "multimodal backend");
expect_error(input, llm_parser, "-backend vlm");
}
TEST_F(PreprocessChatJsonTest, MultipleMessagesWithMixedContent) {
// Multiple messages: some text-only, some with images
// On multimodal, all arrays are preserved (no combining)
std::string input = R"({
"messages": [
{
"role": "system",
"content": [{"type": "text", "text": "You are helpful."}]
},
{
"role": "user",
"content": [
{"type": "image_url", "image_url": {"url": "data:image/png;base64,xyz"}},
{"type": "text", "text": "Describe this image"}
]
}
]
})";
VlmChatJsonParser vlm_parser;
// On multimodal: all arrays preserved unchanged
expect_success(input, vlm_parser, input);
}
// =============================================================================
// Error handling tests
// =============================================================================
TEST_F(PreprocessChatJsonTest, InvalidJsonReturnsError) {
std::string input = "not valid json";
LlmChatJsonParser llm_parser;
expect_error(input, llm_parser, "Invalid JSON");
}
TEST_F(PreprocessChatJsonTest, NonObjectMessageReturnsError) {
std::string input = R"({"messages": ["not an object"]})";
LlmChatJsonParser llm_parser;
expect_error(input, llm_parser, "must be an object");
}
TEST_F(PreprocessChatJsonTest, NonObjectContentItemReturnsError) {
std::string input = R"({
"messages": [{"role": "user", "content": ["not an object"]}]
})";
LlmChatJsonParser llm_parser;
expect_error(input, llm_parser, "must be an object");
}
TEST_F(PreprocessChatJsonTest, MissingTextFieldReturnsError) {
std::string input = R"({
"messages": [{"role": "user", "content": [{"type": "text"}]}]
})";
LlmChatJsonParser llm_parser;
expect_error(input, llm_parser, "Missing or invalid 'text' field");
}
TEST_F(PreprocessChatJsonTest, NonStringTextFieldReturnsError) {
std::string input = R"({
"messages": [{"role": "user", "content": [{"type": "text", "text": 123}]}]
})";
LlmChatJsonParser llm_parser;
expect_error(input, llm_parser, "Missing or invalid 'text' field");
}
TEST_F(PreprocessChatJsonTest, MalformedTextInMultimodalContent) {
// Multimodal mode skips parsing entirely - validation happens downstream
std::string input = R"({
"messages": [{
"role": "user",
"content": [
{"type": "image_url", "image_url": {"url": "..."}},
{"type": "text"}
]
}]
})";
VlmChatJsonParser vlm_parser;
// Should pass through unchanged without validation
expect_success(input, vlm_parser, input);
}
// =============================================================================
// Edge cases
// =============================================================================
TEST_F(PreprocessChatJsonTest, EmptyContentArray) {
// Empty content array - should result in empty string for non-multimodal
std::string input = R"({
"messages": [{"role": "user", "content": []}]
})";
std::string expected = R"({
"messages": [{"role": "user", "content": ""}]
})";
LlmChatJsonParser llm_parser;
VlmChatJsonParser vlm_parser;
expect_success(input, llm_parser, expected);
// For multimodal, empty array is preserved
expect_success(input, vlm_parser, input);
}
TEST_F(PreprocessChatJsonTest, PreservesOtherFields) {
// Other fields in the request should be preserved
std::string input = R"({
"model": "test-model",
"messages": [{"role": "user", "content": [{"type": "text", "text": "Hi"}]}],
"temperature": 0.7,
"max_tokens": 100
})";
std::string expected = R"({
"model": "test-model",
"messages": [{"role": "user", "content": "Hi"}],
"temperature": 0.7,
"max_tokens": 100
})";
LlmChatJsonParser llm_parser;
VlmChatJsonParser vlm_parser;
expect_success(input, llm_parser, expected);
// For multimodal, array is preserved
expect_success(input, vlm_parser, input);
}
TEST_F(PreprocessChatJsonTest, UnknownContentTypeOnMultimodal) {
// Unknown content types should pass through on multimodal
std::string input = R"({
"messages": [{
"role": "user",
"content": [{"type": "video", "video": {"url": "..."}}]
}]
})";
VlmChatJsonParser vlm_parser;
expect_success(input, vlm_parser, input);
}
TEST_F(PreprocessChatJsonTest, UnknownContentTypeErrorsOnTextOnly) {
// Unknown content types should error on text-only with helpful message
std::string input = R"({
"messages": [{
"role": "user",
"content": [{"type": "video", "video": {"url": "..."}}]
}]
})";
LlmChatJsonParser llm_parser;
expect_error(input, llm_parser, "multimodal backend");
}
// =============================================================================
// Anthropic parser tests
// =============================================================================
TEST_F(PreprocessChatJsonTest, AnthropicStringContentRemapped) {
std::string input = R"({
"messages": [{"role": "user", "content": "Hello"}]
})";
std::string expected = R"({
"messages": [{"role": "user", "content_string": "Hello"}]
})";
AnthropicChatJsonParser parser;
expect_success(input, parser, expected);
}
TEST_F(PreprocessChatJsonTest, AnthropicArrayContentRemapped) {
std::string input = R"({
"messages": [{
"role": "user",
"content": [
{"type": "text", "text": "Hello"},
{"type": "image", "source": {"data": "abc"}}
]
}]
})";
std::string expected = R"({
"messages": [{
"role": "user",
"content_blocks": {
"blocks": [
{"type": "text", "text": "Hello"},
{"type": "image", "source": {"data": "abc"}}
]
}
}]
})";
AnthropicChatJsonParser parser;
expect_success(input, parser, expected);
}
TEST_F(PreprocessChatJsonTest, AnthropicSystemStringRemapped) {
std::string input = R"({
"system": "You are helpful.",
"messages": [{"role": "user", "content": "Hi"}]
})";
std::string expected = R"({
"system_string": "You are helpful.",
"messages": [{"role": "user", "content_string": "Hi"}]
})";
AnthropicChatJsonParser parser;
expect_success(input, parser, expected);
}
TEST_F(PreprocessChatJsonTest, AnthropicSystemArrayRemapped) {
std::string input = R"({
"system": [{"type": "text", "text": "You are helpful."}],
"messages": [{"role": "user", "content": "Hi"}]
})";
std::string expected = R"({
"system_blocks": {"blocks": [{"type": "text", "text": "You are helpful."}]},
"messages": [{"role": "user", "content_string": "Hi"}]
})";
AnthropicChatJsonParser parser;
expect_success(input, parser, expected);
}
TEST_F(PreprocessChatJsonTest, AnthropicNoContentNoSystem) {
std::string input = R"({"model": "claude-3"})";
AnthropicChatJsonParser parser;
expect_success(input, parser, input);
}
TEST_F(PreprocessChatJsonTest, AnthropicInvalidJsonReturnsError) {
std::string input = "not valid json";
AnthropicChatJsonParser parser;
expect_error(input, parser, "Invalid JSON");
}
TEST_F(PreprocessChatJsonTest, AnthropicPreservesOtherFields) {
std::string input = R"({
"model": "claude-3",
"max_tokens": 1024,
"messages": [{"role": "user", "content": "Hello"}]
})";
std::string expected = R"({
"model": "claude-3",
"max_tokens": 1024,
"messages": [{"role": "user", "content_string": "Hello"}]
})";
AnthropicChatJsonParser parser;
expect_success(input, parser, expected);
}
} // namespace xllm

300
upstream_ref/xllm/tests/api_service/openai_service_test.cpp Normal file
View File

@@ -0,0 +1,300 @@
/* Copyright 2026 The xLLM Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
https://github.com/jd-opensource/xllm/blob/main/LICENSE
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
// Testing steps:
// 1. start the xllm server first
// 2. run the test with disabled cases enabled
// XLLM_TEST_BASE_URL=http://127.0.0.1:9977 XLLM_TEST_MODEL=Qwen3-8B
// ./build/lib.linux-aarch64-cpython-311/xllm/openai_service_test
// --gtest_also_run_disabled_tests
#include <brpc/channel.h>
#include <brpc/controller.h>
#include <glog/logging.h>
#include <gtest/gtest.h>
#include <cstdlib>
#include <map>
#include <nlohmann/json.hpp>
#include <string>
#include <utility>
namespace xllm {
namespace {
struct TestConfig {
std::string base_url;
std::string model;
std::string api_key;
static TestConfig get() {
TestConfig config;
const char* url_env = std::getenv("XLLM_TEST_BASE_URL");
config.base_url = url_env ? url_env : "http://127.0.0.1:9977";
const char* model_env = std::getenv("XLLM_TEST_MODEL");
config.model = model_env ? model_env : "my_model";
const char* key_env = std::getenv("XLLM_TEST_API_KEY");
config.api_key = key_env ? key_env : "xllm-test-123456";
return config;
}
};
struct HttpResult {
bool controller_failed = false;
int status_code = 0;
std::string content_type;
std::string error_text;
std::string body;
nlohmann::json json = nullptr;
};
class HttpClient {
public:
explicit HttpClient(const std::string& base_url) {
brpc::ChannelOptions options;
options.protocol = brpc::PROTOCOL_HTTP;
options.connection_type = brpc::CONNECTION_TYPE_POOLED;
options.timeout_ms = 60000;
options.max_retry = 3;
if (channel_.Init(base_url.c_str(), &options) != 0) {
LOG(ERROR) << "Failed to init channel for " << base_url;
}
}
HttpResult post(const std::string& path,
const nlohmann::json& body,
const std::map<std::string, std::string>& headers) {
brpc::Controller cntl;
cntl.http_request().uri() = path;
cntl.http_request().set_method(brpc::HTTP_METHOD_POST);
cntl.http_request().set_content_type("application/json");
for (const auto& [key, value] : headers) {
cntl.http_request().SetHeader(key, value);
}
cntl.request_attachment().append(body.dump());
channel_.CallMethod(nullptr, &cntl, nullptr, nullptr, nullptr);
HttpResult result;
result.controller_failed = cntl.Failed();
result.status_code = cntl.http_response().status_code();
result.content_type = cntl.http_response().content_type();
result.error_text = cntl.ErrorText();
result.body = cntl.response_attachment().to_string();
if (!result.body.empty()) {
try {
result.json = nlohmann::json::parse(result.body);
} catch (const std::exception&) {
result.json = nullptr;
}
}
return result;
}
private:
brpc::Channel channel_;
};
std::string describe_result(const HttpResult& result) {
std::string description = "status=" + std::to_string(result.status_code);
if (!result.content_type.empty()) {
description += ", content_type=" + result.content_type;
}
if (!result.error_text.empty()) {
description += ", error=" + result.error_text;
}
if (!result.body.empty()) {
description += ", body=" + result.body;
}
return description;
}
void expect_error_contains(const HttpResult& result,
const std::string& expected_fragment) {
const std::string haystack =
result.error_text + "\n" + result.body + "\n" + result.json.dump();
EXPECT_NE(haystack.find(expected_fragment), std::string::npos)
<< "Expected fragment '" << expected_fragment
<< "' in response, got: " << describe_result(result);
}
void expect_logprobs_shape(const nlohmann::json& choice) {
ASSERT_TRUE(choice.contains("logprobs"));
ASSERT_TRUE(choice["logprobs"].is_object());
ASSERT_TRUE(choice["logprobs"].contains("tokens"));
ASSERT_TRUE(choice["logprobs"].contains("token_ids"));
ASSERT_TRUE(choice["logprobs"].contains("token_logprobs"));
ASSERT_TRUE(choice["logprobs"]["tokens"].is_array());
ASSERT_TRUE(choice["logprobs"]["token_ids"].is_array());
ASSERT_TRUE(choice["logprobs"]["token_logprobs"].is_array());
EXPECT_EQ(choice["logprobs"]["tokens"].size(),
choice["logprobs"]["token_ids"].size());
EXPECT_EQ(choice["logprobs"]["tokens"].size(),
choice["logprobs"]["token_logprobs"].size());
}
class DISABLED_OpenAIServerFeaturesTest : public ::testing::Test {
protected:
void SetUp() override {
config_ = TestConfig::get();
client_ = std::make_unique<HttpClient>(config_.base_url);
}
std::map<std::string, std::string> get_headers() const {
return {{"Authorization", "Bearer " + config_.api_key},
{"Content-Type", "application/json"}};
}
nlohmann::json make_sample_request(const std::string& prompt,
int logprobs = 3) const {
return {{"model", config_.model},
{"prompt", prompt},
{"selector", {{"type", "literal"}, {"value", "<emb_0>"}}},
{"logprobs", logprobs},
{"request_id", "sample-it"}};
}
void expect_sample_choice(const nlohmann::json& choice,
std::size_t expected_index,
std::size_t max_logprobs) const {
EXPECT_EQ(choice["index"], expected_index);
ASSERT_TRUE(choice.contains("finish_reason"));
ASSERT_TRUE(choice.contains("text"));
expect_logprobs_shape(choice);
EXPECT_LE(choice["logprobs"]["tokens"].size(), max_logprobs);
const std::string finish_reason = choice["finish_reason"];
EXPECT_TRUE(finish_reason == "selector_match" ||
finish_reason == "empty_logprobs")
<< "Unexpected finish_reason: " << finish_reason;
if (!choice["logprobs"]["tokens"].empty()) {
EXPECT_EQ(choice["text"], choice["logprobs"]["tokens"][0]);
} else {
EXPECT_TRUE(choice["text"].empty());
EXPECT_EQ(finish_reason, "empty_logprobs");
}
}
TestConfig config_;
std::unique_ptr<HttpClient> client_;
};
TEST_F(DISABLED_OpenAIServerFeaturesTest, SampleSingleMatch) {
const HttpResult result =
client_->post("/v1/sample",
make_sample_request("Classify <emb_0> in one token."),
get_headers());
ASSERT_FALSE(result.controller_failed) << describe_result(result);
ASSERT_EQ(result.status_code, 200) << describe_result(result);
EXPECT_EQ(result.content_type, "application/json") << describe_result(result);
ASSERT_TRUE(result.json.is_object()) << describe_result(result);
EXPECT_EQ(result.json["id"], "sample-it");
EXPECT_EQ(result.json["object"], "sample_completion");
EXPECT_EQ(result.json["model"], config_.model);
ASSERT_TRUE(result.json.contains("choices"));
ASSERT_EQ(result.json["choices"].size(), 1);
expect_sample_choice(result.json["choices"][0], 0, 3);
}
TEST_F(DISABLED_OpenAIServerFeaturesTest, SampleMultipleMatchesStayOrdered) {
const HttpResult result = client_->post(
"/v1/sample", make_sample_request("A<emb_0>B<emb_0>C"), get_headers());
ASSERT_FALSE(result.controller_failed) << describe_result(result);
ASSERT_EQ(result.status_code, 200) << describe_result(result);
ASSERT_TRUE(result.json.is_object()) << describe_result(result);
ASSERT_TRUE(result.json.contains("choices"));
ASSERT_EQ(result.json["choices"].size(), 2);
expect_sample_choice(result.json["choices"][0], 0, 3);
expect_sample_choice(result.json["choices"][1], 1, 3);
}
TEST_F(DISABLED_OpenAIServerFeaturesTest,
SampleSelectorMissReturnsEmptyChoices) {
const HttpResult result = client_->post(
"/v1/sample", make_sample_request("plain text"), get_headers());
ASSERT_FALSE(result.controller_failed) << describe_result(result);
ASSERT_EQ(result.status_code, 200) << describe_result(result);
ASSERT_TRUE(result.json.is_object()) << describe_result(result);
EXPECT_EQ(result.json["id"], "sample-it");
EXPECT_EQ(result.json["object"], "sample_completion");
ASSERT_TRUE(result.json.contains("choices"));
EXPECT_TRUE(result.json["choices"].empty());
}
TEST_F(DISABLED_OpenAIServerFeaturesTest,
SampleRejectsUnsupportedSelectorType) {
nlohmann::json request = make_sample_request("A<emb_0>");
request["selector"]["type"] = "regex";
const HttpResult result = client_->post("/v1/sample", request, get_headers());
EXPECT_NE(result.status_code, 200) << describe_result(result);
expect_error_contains(result, "literal");
}
TEST_F(DISABLED_OpenAIServerFeaturesTest, SampleRejectsOutOfRangeLogprobs) {
nlohmann::json request = make_sample_request("A<emb_0>");
request["logprobs"] = 0;
const HttpResult result = client_->post("/v1/sample", request, get_headers());
EXPECT_NE(result.status_code, 200) << describe_result(result);
expect_error_contains(result, "between 1 and 5");
}
TEST_F(DISABLED_OpenAIServerFeaturesTest, CompletionsRegressionSmoke) {
nlohmann::json request = {{"model", config_.model},
{"prompt", "Say hi."},
{"max_tokens", 1},
{"temperature", 0.0}};
const HttpResult result =
client_->post("/v1/completions", request, get_headers());
ASSERT_FALSE(result.controller_failed) << describe_result(result);
ASSERT_EQ(result.status_code, 200) << describe_result(result);
EXPECT_EQ(result.content_type, "application/json") << describe_result(result);
ASSERT_TRUE(result.json.is_object()) << describe_result(result);
ASSERT_TRUE(result.json.contains("choices"));
ASSERT_EQ(result.json["choices"].size(), 1);
EXPECT_EQ(result.json["choices"][0]["index"], 0);
ASSERT_TRUE(result.json.contains("usage"));
}
TEST_F(DISABLED_OpenAIServerFeaturesTest, ChatCompletionsRegressionSmoke) {
nlohmann::json request = {
{"model", config_.model},
{"messages", {{{"role", "user"}, {"content", "Say hi."}}}},
{"max_tokens", 1},
{"temperature", 0.0}};
const HttpResult result =
client_->post("/v1/chat/completions", request, get_headers());
ASSERT_FALSE(result.controller_failed) << describe_result(result);
ASSERT_EQ(result.status_code, 200) << describe_result(result);
EXPECT_EQ(result.content_type, "application/json") << describe_result(result);
ASSERT_TRUE(result.json.is_object()) << describe_result(result);
ASSERT_TRUE(result.json.contains("choices"));
ASSERT_EQ(result.json["choices"].size(), 1);
ASSERT_TRUE(result.json["choices"][0].contains("message"));
ASSERT_TRUE(result.json["choices"][0]["message"].contains("role"));
ASSERT_TRUE(result.json["choices"][0]["message"].contains("content"));
ASSERT_TRUE(result.json.contains("usage"));
}
} // namespace
} // namespace xllm

335
upstream_ref/xllm/tests/api_service/sample_service_impl_test.cpp Normal file
View File

@@ -0,0 +1,335 @@
/* Copyright 2026 The xLLM Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
https://github.com/jd-opensource/xllm/blob/main/LICENSE
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#include "sample_service_impl.h"
#include <gtest/gtest.h>
#include <cstdint>
#include <cstring>
#include <string>
#include <string_view>
#include <vector>
#include "core/framework/tokenizer/tokenizer.h"
namespace xllm {
namespace {
class CharTokenizer final : public Tokenizer {
public:
bool encode(const std::string_view& text,
std::vector<int32_t>* ids,
bool add_special_tokens = true) const override {
if (ids == nullptr) {
return false;
}
ids->clear();
if (add_special_tokens) {
ids->push_back(kBosTokenId);
}
size_t pos = 0;
while (pos < text.size()) {
if (text.size() - pos >= kEmbTokenLen &&
std::memcmp(text.data() + pos, kEmbToken, kEmbTokenLen) == 0) {
ids->push_back(kEmbTokenId);
pos += kEmbTokenLen;
continue;
}
ids->push_back(static_cast<unsigned char>(text[pos]));
++pos;
}
return true;
}
std::optional<int32_t> token_to_id(
const std::string_view& token) const override {
if (token == std::string_view(kEmbToken, kEmbTokenLen)) {
return kEmbTokenId;
}
return std::nullopt;
}
std::string id_to_token(int32_t id) const override {
if (id == kBosTokenId) {
return "<bos>";
}
if (id == kEmbTokenId) {
return std::string(kEmbToken, kEmbTokenLen);
}
return std::string(1, static_cast<char>(id));
}
private:
static constexpr int32_t kBosTokenId = 1;
static constexpr int32_t kEmbTokenId = 100000;
static constexpr char kEmbToken[] = "<emb_0>";
static constexpr size_t kEmbTokenLen = sizeof(kEmbToken) - 1;
};
class UnstableLiteralTokenizer final : public Tokenizer {
public:
bool encode(const std::string_view& text,
std::vector<int32_t>* ids,
bool add_special_tokens = true) const override {
if (ids == nullptr) {
return false;
}
ids->clear();
if (add_special_tokens) {
ids->push_back(1);
}
for (char ch : text) {
ids->push_back(static_cast<unsigned char>(ch));
}
return true;
}
};
proto::SampleRequest make_valid_request() {
proto::SampleRequest request;
request.set_model("mtp");
request.set_prompt("plain text");
auto* selector = request.mutable_selector();
selector->set_type("literal");
selector->set_value("<emb_0>");
return request;
}
TEST(SampleServiceImplTest, ValidateRequestRejectsUnsupportedSelectorType) {
auto request = make_valid_request();
request.mutable_selector()->set_type("regex");
const Status status = sample_service_internal::validate_request(request);
EXPECT_EQ(status.code(), StatusCode::INVALID_ARGUMENT);
EXPECT_NE(status.message().find("literal"), std::string::npos);
}
TEST(SampleServiceImplTest, ValidateRequestRejectsOutOfRangeLogprobs) {
auto request = make_valid_request();
request.set_logprobs(0);
Status status = sample_service_internal::validate_request(request);
EXPECT_EQ(status.code(), StatusCode::INVALID_ARGUMENT);
EXPECT_NE(status.message().find("between 1 and 5"), std::string::npos);
request.set_logprobs(6);
status = sample_service_internal::validate_request(request);
EXPECT_EQ(status.code(), StatusCode::INVALID_ARGUMENT);
EXPECT_NE(status.message().find("between 1 and 5"), std::string::npos);
}
TEST(SampleServiceImplTest, ValidateRequestRejectsMissingRequiredFields) {
auto request = make_valid_request();
request.clear_model();
Status status = sample_service_internal::validate_request(request);
EXPECT_EQ(status.code(), StatusCode::INVALID_ARGUMENT);
EXPECT_EQ(status.message(), "model is required");
request = make_valid_request();
request.clear_prompt();
status = sample_service_internal::validate_request(request);
EXPECT_EQ(status.code(), StatusCode::INVALID_ARGUMENT);
EXPECT_EQ(status.message(), "prompt is required");
request = make_valid_request();
request.clear_selector();
status = sample_service_internal::validate_request(request);
EXPECT_EQ(status.code(), StatusCode::INVALID_ARGUMENT);
EXPECT_EQ(status.message(), "selector is required");
request = make_valid_request();
request.mutable_selector()->clear_value();
status = sample_service_internal::validate_request(request);
EXPECT_EQ(status.code(), StatusCode::INVALID_ARGUMENT);
EXPECT_EQ(status.message(), "selector.value is required");
}
TEST(SampleServiceImplTest, ValidateRuntimeConfigRejectsScheduleOverlap) {
Status status = sample_service_internal::validate_runtime_config(true);
EXPECT_EQ(status.code(), StatusCode::UNAVAILABLE);
EXPECT_NE(status.message().find("does not support async scheduling"),
std::string::npos);
status = sample_service_internal::validate_runtime_config(false);
EXPECT_TRUE(status.ok());
}
TEST(SampleServiceImplTest, BuildRequestParamsAndEmptyResponseForSelectorMiss) {
CharTokenizer tokenizer;
auto request = make_valid_request();
RequestParams params;
ASSERT_TRUE(sample_service_internal::build_request_params(
request, tokenizer, &params));
EXPECT_TRUE(params.logprobs);
EXPECT_EQ(params.top_logprobs,
sample_service_internal::kDefaultSampleLogprobs);
EXPECT_TRUE(params.is_sample_request);
EXPECT_TRUE(params.sample_slots.empty());
EXPECT_FALSE(params.request_id.empty());
proto::SampleResponse response;
ASSERT_TRUE(sample_service_internal::build_empty_response(
request, tokenizer, params.request_id, &response));
EXPECT_EQ(response.id(), params.request_id);
EXPECT_EQ(response.object(), "sample_completion");
EXPECT_EQ(response.model(), request.model());
EXPECT_EQ(response.choices_size(), 0);
EXPECT_GT(response.created(), 0U);
ASSERT_TRUE(response.has_usage());
const int32_t expected_prompt_tokens =
static_cast<int32_t>(request.prompt().size() + 1);
EXPECT_EQ(response.usage().prompt_tokens(), expected_prompt_tokens);
EXPECT_EQ(response.usage().completion_tokens(), 0);
EXPECT_EQ(response.usage().total_tokens(), expected_prompt_tokens);
}
TEST(SampleServiceImplTest,
BuildRequestParamsKeepsExplicitRequestIdAndMatchedSampleSlots) {
CharTokenizer tokenizer;
auto request = make_valid_request();
request.set_prompt("A<emb_0>B<emb_0>C");
request.set_request_id("sample-explicit");
request.set_logprobs(4);
RequestParams params;
ASSERT_TRUE(sample_service_internal::build_request_params(
request, tokenizer, &params));
EXPECT_EQ(params.request_id, "sample-explicit");
EXPECT_TRUE(params.logprobs);
EXPECT_EQ(params.top_logprobs, 4);
EXPECT_TRUE(params.is_sample_request);
ASSERT_EQ(params.sample_slots.size(), 2);
EXPECT_EQ(params.sample_slots[0].request_id, "sample-explicit");
EXPECT_EQ(params.sample_slots[0].sample_id, 0);
EXPECT_EQ(params.sample_slots[0].token_position, 1);
EXPECT_EQ(params.sample_slots[1].request_id, "sample-explicit");
EXPECT_EQ(params.sample_slots[1].sample_id, 1);
EXPECT_EQ(params.sample_slots[1].token_position, 3);
}
TEST(SampleServiceImplTest, BuildRequestParamsRejectsUnstableLiteralToken) {
UnstableLiteralTokenizer tokenizer;
auto request = make_valid_request();
request.set_prompt("A<emb_0>B");
request.set_request_id("sample-explicit");
RequestParams params;
EXPECT_FALSE(sample_service_internal::build_request_params(
request, tokenizer, &params));
}
TEST(SampleServiceImplTest,
BuildResponseSortsBySampleIdAndSerializesTopLogprobs) {
RequestOutput req_output;
Usage usage;
usage.num_prompt_tokens = 8;
usage.num_generated_tokens = 2;
usage.num_total_tokens = 10;
req_output.usage = usage;
SequenceOutput missing_output;
missing_output.index = 1;
missing_output.finish_reason = "empty_logprobs";
SequenceOutput sampled_output;
sampled_output.index = 0;
sampled_output.text = "stale";
LogProb sampled_logprob;
sampled_logprob.token = "True";
sampled_logprob.token_id = 101;
sampled_logprob.logprob = -0.10f;
std::vector<LogProbData> top_logprobs;
LogProbData top1;
top1.token = "True";
top1.token_id = 101;
top1.logprob = -0.10f;
top_logprobs.push_back(top1);
LogProbData top2;
top2.token = "False";
top2.token_id = 102;
top2.logprob = -2.30f;
top_logprobs.push_back(top2);
sampled_logprob.top_logprobs = top_logprobs;
sampled_output.logprobs = std::vector<LogProb>{sampled_logprob};
req_output.outputs = {missing_output, sampled_output};
proto::SampleResponse response;
ASSERT_TRUE(sample_service_internal::build_response(
"sample-123", "mtp", 1773369600U, req_output, &response));
EXPECT_EQ(response.id(), "sample-123");
EXPECT_EQ(response.object(), "sample_completion");
EXPECT_EQ(response.created(), 1773369600U);
EXPECT_EQ(response.model(), "mtp");
ASSERT_EQ(response.choices_size(), 2);
EXPECT_EQ(response.choices(0).index(), 0);
EXPECT_EQ(response.choices(0).text(), "True");
EXPECT_EQ(response.choices(0).finish_reason(), "selector_match");
ASSERT_TRUE(response.choices(0).has_logprobs());
EXPECT_EQ(response.choices(0).logprobs().tokens_size(), 2);
EXPECT_EQ(response.choices(0).logprobs().tokens(0), "True");
EXPECT_EQ(response.choices(0).logprobs().tokens(1), "False");
EXPECT_EQ(response.choices(0).logprobs().token_ids(0), 101);
EXPECT_EQ(response.choices(0).logprobs().token_ids(1), 102);
EXPECT_EQ(response.choices(1).index(), 1);
EXPECT_TRUE(response.choices(1).text().empty());
EXPECT_EQ(response.choices(1).finish_reason(), "empty_logprobs");
ASSERT_TRUE(response.choices(1).has_logprobs());
EXPECT_EQ(response.choices(1).logprobs().tokens_size(), 0);
EXPECT_EQ(response.choices(1).logprobs().token_ids_size(), 0);
EXPECT_EQ(response.choices(1).logprobs().token_logprobs_size(), 0);
ASSERT_TRUE(response.has_usage());
EXPECT_EQ(response.usage().prompt_tokens(), 8);
EXPECT_EQ(response.usage().completion_tokens(), 2);
EXPECT_EQ(response.usage().total_tokens(), 10);
}
TEST(SampleServiceImplTest, BuildResponseFallsBackToSelectedTokenLogprob) {
RequestOutput req_output;
SequenceOutput sampled_output;
sampled_output.index = 0;
LogProb sampled_logprob;
sampled_logprob.token = "Maybe";
sampled_logprob.token_id = 201;
sampled_logprob.logprob = -0.25f;
sampled_output.logprobs = std::vector<LogProb>{sampled_logprob};
req_output.outputs = {sampled_output};
proto::SampleResponse response;
ASSERT_TRUE(sample_service_internal::build_response(
"sample-456", "mtp", 1773369601U, req_output, &response));
ASSERT_EQ(response.choices_size(), 1);
EXPECT_EQ(response.choices(0).text(), "Maybe");
EXPECT_EQ(response.choices(0).finish_reason(), "selector_match");
ASSERT_TRUE(response.choices(0).has_logprobs());
EXPECT_EQ(response.choices(0).logprobs().tokens_size(), 1);
EXPECT_EQ(response.choices(0).logprobs().tokens(0), "Maybe");
EXPECT_EQ(response.choices(0).logprobs().token_ids(0), 201);
}
} // namespace
} // namespace xllm