dylanyunlong/project_6_89d52222
1
0
Fork 0
Code Issues Pull Requests Actions Projects Releases Wiki Activity

ref(upstream): FULL TREE — Deep-Spark xllm (1470) + ds_vllm csrc/models (703)

Browse Source 
Replaces cherry-picked upstream_ref with complete source trees.

xllm/ — Iluvatar official C++ inference engine (15MB, 1470 files)
  Complete: kernels → layers → models → runtime → scheduler → api
  Excluded: .git, binary images, third_party submodule checkouts

ds_vllm/ — Iluvatar official vllm fork (8MB, 703 files)
  Included: csrc/ (ALL CUDA kernels), fused_moe/, qwen3_5 model, _custom_ops
  Excluded: tests, benchmarks, docs, examples (not needed for reference)

Critical call chains now fully traceable:
  MoE: moe_topk_softmax_kernels.cuh → ixformer.h → fused_moe.cpp → layer
  GDN: qwen3_gated_delta_net_base.cpp → qwen3_5_gated_delta_net.cpp
  Attention: ixformer.h → xllm_paged_attention → attention.cpp
This commit is contained in:
EX Engine
2026-08-10 02:53:54 +00:00
parent 9e4fb3712f
commit 002f9879b2
2179 changed files with 494021 additions and 79 deletions
Download Patch File Download Diff File Expand all files Collapse all files

104
upstream_ref/xllm/xllm/c_api/README.md Normal file
View File

@@ -0,0 +1,104 @@
### How to compile xllm dynamic library
Run the following command in root directory:
```
python setup.py build --generate-so true
```
If you want to debug, it needs to set DEBUG environment variable.
```
export DEBUG=1
```
### How to install dynamic library
Run installation script xllm/c_api/install.sh, headers and dynamic library will be installed in /usr/local/xllm directory.
```
cd xllm/c_api/tools
sh install.sh
```
You will see the following files in /usr/local/xllm directory:
```
[root@A03-R40-I189-101-4100046]# tree /usr/local/xllm
/usr/local/xllm
|-- include
| |-- llm.h
| |-- default.h
| |-- rec.h
| `-- types.h
`-- lib
`-- libxllm.so
3 directories, 5 files
```
### How to compile c_api examples
GPU builds and NPU builds use different link commands. Replace
`<example>.cpp` and `<example>` with the example source file and output binary
name you want to build.
#### GPU
```
cd xllm/c_api/examples
g++ <example>.cpp -o <example> \
-std=c++17 \
-DUSE_CUDA \
-I/usr/local/xllm/include \
-L/usr/local/xllm/lib \
-lxllm \
-Wl,-rpath=/usr/local/xllm/lib
```
#### NPU
Before compiling or running examples, source the Ascend environment first:
```
source /usr/local/Ascend/ascend-toolkit/set_env.sh
```
Then compile with the extra custom op library used by the NPU build:
```
cd xllm/c_api/examples
g++ <example>.cpp -o <example> \
-std=c++17 \
-DUSE_NPU \
-I/usr/local/xllm/include \
-L/usr/local/xllm/lib \
-L/usr/local/Ascend/ascend-toolkit/latest/opp/vendors/xllm/op_api/lib \
-lxllm \
-lcust_opapi \
-Wl,-rpath=/usr/local/xllm/lib \
-Wl,-rpath=/usr/local/Ascend/ascend-toolkit/latest/opp/vendors/xllm/op_api/lib
```
If `-lcust_opapi` is missing from the NPU link command, the linker may report
undefined references to symbols such as `aclnnBeamSearchGroup` and
`aclnnXAttention`.
### How to run c_api examples
Some examples, such as `simple_rec_completions`, support overriding the target
device from `argv[1]`.
#### NPU
```
./simple_rec_completions npu:14
```
#### GPU
```
./simple_rec_completions cuda:0
```

161
upstream_ref/xllm/xllm/c_api/default.h Normal file
View File

@@ -0,0 +1,161 @@
/* Copyright 2025 The xLLM Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
https://github.com/jd-opensource/xllm/blob/main/LICENSE
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef XLLM_LLM_DEFAULT_H
#define XLLM_LLM_DEFAULT_H
#ifdef __cplusplus
extern "C" {
#endif
#include "types.h"
const XLLM_InitOptions XLLM_INIT_LLM_OPTIONS_DEFAULT = {
.enable_chunked_prefill = false,
.enable_prefill_sp = false,
.enable_prefix_cache = false,
.enable_disagg_pd = false,
.enable_pd_ooc = false,
.enable_schedule_overlap = false,
.enable_shm = false,
.transfer_listen_port = 26000,
.nnodes = 1,
.node_rank = 0,
.dp_size = 1,
.ep_size = 1,
.block_size = 32,
.max_cache_size = 0,
.max_tokens_per_batch = 20480,
.max_seqs_per_batch = 256,
.max_tokens_per_chunk_for_prefill = 0,
.num_speculative_tokens = 0,
.num_request_handling_threads = 4,
.expert_parallel_degree = 0,
.server_idx = 0,
.max_memory_utilization = 0.9,
.task = "generate",
.communication_backend = "lccl",
.instance_role = "DEFAULT",
.device_ip = "",
.master_node_addr = "127.0.0.1:18899",
.xservice_addr = "",
.instance_name = "",
.kv_cache_transfer_mode = "PUSH",
.log_dir = "",
.draft_model = "",
.draft_devices = ""};
const XLLM_RequestParams XLLM_LLM_REQUEST_PARAMS_DEFAULT = {
.echo = false,
.offline = false,
.logprobs = false,
.ignore_eos = false,
.n = 1,
.max_tokens = 5120,
.best_of = 1,
.ttlt_slo_ms = INT32_MAX,
.ttft_slo_ms = INT32_MAX,
.tpot_slo_ms = INT32_MAX,
.beam_width = 0,
.num_return_sequences = 0,
.top_logprobs = 0,
.top_k = -1,
.top_p = 1.0,
.frequency_penalty = 0.0,
.presence_penalty = 0.0,
.repetition_penalty = 1.0,
.temperature = 0.0,
.request_id = ""};
const XLLM_InitOptions XLLM_INIT_REC_OPTIONS_DEFAULT = {
.enable_chunked_prefill = false,
.enable_prefill_sp = false,
.enable_prefix_cache = false,
.enable_disagg_pd = false,
.enable_pd_ooc = false,
.enable_schedule_overlap = false,
.enable_shm = false,
.enable_graph = true,
.enable_rec_fast_sampler = true,
.enable_prefill_piecewise_graph = true,
.enable_xattention_one_stage = false,
.enable_graph_mode_decode_no_padding = true,
.enable_block_copy_kernel = false,
.enable_topk_sorted = false,
.enable_rec_prefill_only = false,
.transfer_listen_port = 26000,
.nnodes = 1,
.node_rank = 0,
.dp_size = 1,
.ep_size = 1,
.block_size = 1,
.max_cache_size = 1000000,
.max_tokens_per_batch = 4096,
.max_seqs_per_batch = 4,
.max_tokens_per_chunk_for_prefill = 0,
.num_speculative_tokens = 0,
.num_request_handling_threads = 4,
.expert_parallel_degree = 0,
.server_idx = 0,
.beam_width = 128,
.max_decode_rounds = 3,
.max_token_per_req = 1000,
.max_memory_utilization = 0.55,
.rec_worker_max_concurrency = 2,
.task = "generate",
.communication_backend = "lccl",
.instance_role = "DEFAULT",
.device_ip = "",
.master_node_addr = "127.0.0.1:18899",
.xservice_addr = "",
.instance_name = "",
.kv_cache_transfer_mode = "PUSH",
.log_dir = "",
.draft_model = "",
.draft_devices = ""};
const XLLM_RequestParams XLLM_REC_REQUEST_PARAMS_DEFAULT = {
.echo = false,
.offline = false,
.logprobs = false,
.ignore_eos = false,
.n = 1,
.max_tokens = 5120,
.best_of = 1,
.ttlt_slo_ms = INT32_MAX,
.ttft_slo_ms = INT32_MAX,
.tpot_slo_ms = INT32_MAX,
.beam_width = 128,
.num_return_sequences = 0,
.top_logprobs = 0,
.top_k = -1,
.top_p = 1.0,
.frequency_penalty = 0.0,
.presence_penalty = 0.0,
.repetition_penalty = 1.0,
.temperature = 0.0,
.request_id = ""};
#ifdef __cplusplus
}
#endif
#endif // XLLM_LLM_DEFAULT_H

956
upstream_ref/xllm/xllm/c_api/examples/test_press_rec_multimodal_completions.cpp Normal file
View File

@@ -0,0 +1,956 @@
/* Copyright 2026 The xLLM Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
https://github.com/jd-opensource/xllm/blob/main/LICENSE
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#include <unistd.h>
#include <algorithm>
#include <atomic>
#include <chrono>
#include <cmath>
#include <condition_variable>
#include <cstdint>
#include <cstdlib>
#include <cstring>
#include <deque>
#include <filesystem>
#include <fstream>
#include <iomanip>
#include <iostream>
#include <limits>
#include <memory>
#include <mutex>
#include <optional>
#include <random>
#include <sstream>
#include <stdexcept>
#include <string>
#include <thread>
#include <utility>
#include <vector>
#include "rec.h"
namespace {
using Clock = std::chrono::steady_clock;
constexpr int32_t kFixedRequestMaxTokens = 3;
constexpr int32_t kFixedRequestBeamWidth = 128;
constexpr int32_t kFixedRequestTopK = 128;
constexpr int32_t kFixedRequestTopLogprobs = 128;
constexpr bool kFixedRequestLogprobs = true;
constexpr int32_t kFixedMmItemsPerRequest = 2;
constexpr int32_t kMaxClientParallelism = 128;
struct CliOptions {
std::string model_path;
std::string devices = "cuda:0";
std::string master_node_addr = "127.0.0.1:18899";
int32_t prompt_size = 128;
int32_t token_min_size = 1024;
int32_t token_max_size = 1024;
double qps = 1.0;
int32_t duration_s = 60;
int32_t client_threads = 0;
int32_t timeout_ms = 30000;
int32_t mm_min_span = 8;
int32_t mm_max_span = 64;
uint32_t seed = 20260410U;
};
struct ModelConfig {
int32_t hidden_size = 0;
int32_t vocab_size = 0;
int32_t max_position_embeddings = 0;
std::string model_type;
};
struct Metrics {
std::atomic<uint64_t> sent{0};
std::atomic<uint64_t> succeeded{0};
std::atomic<uint64_t> failed{0};
std::atomic<uint64_t> timeout{0};
std::atomic<uint64_t> invalid_request{0};
std::atomic<uint64_t> internal_error{0};
std::atomic<uint64_t> total_prompt_tokens{0};
std::atomic<uint64_t> total_completion_tokens{0};
std::atomic<uint64_t> total_latency_us{0};
std::atomic<uint64_t> max_latency_us{0};
std::atomic<uint64_t> current_in_flight{0};
std::atomic<uint64_t> max_in_flight{0};
};
struct RequestPayload {
std::vector<int32_t> token_ids;
};
struct ScheduledRequest {
uint64_t request_index = 0;
size_t pool_index = 0;
};
class EmbeddingMmDataBuilder {
public:
EmbeddingMmDataBuilder() = default;
const XLLM_MM_Data* Build(
const std::vector<std::pair<uint32_t, uint32_t>>& spans,
int32_t hidden_size,
uint64_t request_index,
std::mt19937& rng) {
Reset();
if (spans.empty()) {
return nullptr;
}
mm_data_.type_mask = static_cast<uint32_t>(XLLM_MM_TYPE_EMBEDDING);
mm_data_.is_dict = false;
items_.reserve(spans.size());
buffers_.reserve(spans.size());
for (size_t item_idx = 0; item_idx < spans.size(); ++item_idx) {
const auto [offset, length] = spans[item_idx];
if (length == 0) {
continue;
}
XLLM_MM_Item item{};
item.type = XLLM_MM_TYPE_EMBEDDING;
item.state.token_pos.offset = offset;
item.state.token_pos.length = length;
item.data.is_single_tensor = true;
item.data.data.tensor.dtype = XLLM_DTYPE_BFLOAT16;
item.data.data.tensor.dims.rank = 2;
item.data.data.tensor.dims.dim[0] = static_cast<int>(length);
item.data.data.tensor.dims.dim[1] = hidden_size;
auto buffer = std::make_unique<uint16_t[]>(
static_cast<size_t>(length) * static_cast<size_t>(hidden_size));
FillEmbeddingBuffer(
buffer.get(), length, hidden_size, request_index, item_idx, rng);
item.data.data.tensor.data = buffer.get();
buffers_.push_back(std::move(buffer));
items_.push_back(item);
}
if (items_.empty()) {
return nullptr;
}
mm_data_.data.items.entries = items_.data();
mm_data_.data.items.entries_size = items_.size();
return &mm_data_;
}
private:
static uint16_t FloatToBFloat16(float value) {
union {
float f32;
uint32_t u32;
} bits;
bits.f32 = value;
return static_cast<uint16_t>(bits.u32 >> 16);
}
void FillEmbeddingBuffer(uint16_t* dst,
uint32_t length,
int32_t hidden_size,
uint64_t request_index,
size_t item_idx,
std::mt19937& rng) {
std::uniform_real_distribution<float> dist(-1.0f, 1.0f);
const size_t element_count =
static_cast<size_t>(length) * static_cast<size_t>(hidden_size);
for (size_t element_idx = 0; element_idx < element_count; ++element_idx) {
const float noise = dist(rng) * 0.03125f;
const float base =
std::sin(static_cast<float>((request_index + 1) * 0.013) +
static_cast<float>(item_idx) * 0.17f +
static_cast<float>(element_idx % hidden_size) * 0.001f);
dst[element_idx] = FloatToBFloat16(base + noise);
}
}
void Reset() {
std::memset(&mm_data_, 0, sizeof(mm_data_));
items_.clear();
buffers_.clear();
}
XLLM_MM_Data mm_data_{};
std::vector<XLLM_MM_Item> items_;
std::vector<std::unique_ptr<uint16_t[]>> buffers_;
};
std::string Trim(const std::string& value) {
const auto begin = value.find_first_not_of(" \t\r\n");
if (begin == std::string::npos) {
return "";
}
const auto end = value.find_last_not_of(" \t\r\n");
return value.substr(begin, end - begin + 1);
}
std::string RemoveJsonComments(std::string content) {
std::string output;
output.reserve(content.size());
bool in_string = false;
bool escape = false;
for (size_t i = 0; i < content.size(); ++i) {
const char ch = content[i];
if (in_string) {
output.push_back(ch);
if (escape) {
escape = false;
} else if (ch == '\\') {
escape = true;
} else if (ch == '"') {
in_string = false;
}
continue;
}
if (ch == '"') {
in_string = true;
output.push_back(ch);
continue;
}
if (ch == '/' && i + 1 < content.size()) {
if (content[i + 1] == '/') {
i += 2;
while (i < content.size() && content[i] != '\n') {
++i;
}
if (i < content.size()) {
output.push_back('\n');
}
continue;
}
if (content[i + 1] == '*') {
i += 2;
while (i + 1 < content.size() &&
!(content[i] == '*' && content[i + 1] == '/')) {
++i;
}
++i;
continue;
}
}
output.push_back(ch);
}
return output;
}
const std::string* FindObjectRange(const std::string& content,
const std::string& key,
size_t* object_begin,
size_t* object_end) {
const std::string pattern = "\"" + key + "\"";
const size_t key_pos = content.find(pattern);
if (key_pos == std::string::npos) {
return nullptr;
}
size_t pos = content.find(':', key_pos + pattern.size());
if (pos == std::string::npos) {
return nullptr;
}
++pos;
while (pos < content.size() &&
std::isspace(static_cast<unsigned char>(content[pos]))) {
++pos;
}
if (pos >= content.size() || content[pos] != '{') {
return nullptr;
}
const size_t begin = pos;
int depth = 0;
bool in_string = false;
bool escape = false;
for (; pos < content.size(); ++pos) {
const char ch = content[pos];
if (in_string) {
if (escape) {
escape = false;
} else if (ch == '\\') {
escape = true;
} else if (ch == '"') {
in_string = false;
}
continue;
}
if (ch == '"') {
in_string = true;
continue;
}
if (ch == '{') {
++depth;
} else if (ch == '}') {
--depth;
if (depth == 0) {
*object_begin = begin;
*object_end = pos + 1;
return &content;
}
}
}
return nullptr;
}
std::optional<std::string> FindJsonStringValueInRange(
const std::string& content,
size_t begin,
size_t end,
const std::string& key) {
const std::string pattern = "\"" + key + "\"";
size_t pos = content.find(pattern, begin);
if (pos == std::string::npos || pos >= end) {
return std::nullopt;
}
pos = content.find(':', pos + pattern.size());
if (pos == std::string::npos || pos >= end) {
return std::nullopt;
}
++pos;
while (pos < end && std::isspace(static_cast<unsigned char>(content[pos]))) {
++pos;
}
if (pos >= end || content[pos] != '"') {
return std::nullopt;
}
++pos;
std::string value;
bool escape = false;
while (pos < end) {
const char ch = content[pos++];
if (escape) {
value.push_back(ch);
escape = false;
continue;
}
if (ch == '\\') {
escape = true;
continue;
}
if (ch == '"') {
return value;
}
value.push_back(ch);
}
return std::nullopt;
}
std::optional<int32_t> FindJsonIntValueInRange(const std::string& content,
size_t begin,
size_t end,
const std::string& key) {
const std::string pattern = "\"" + key + "\"";
size_t pos = content.find(pattern, begin);
if (pos == std::string::npos || pos >= end) {
return std::nullopt;
}
pos = content.find(':', pos + pattern.size());
if (pos == std::string::npos || pos >= end) {
return std::nullopt;
}
++pos;
while (pos < end && std::isspace(static_cast<unsigned char>(content[pos]))) {
++pos;
}
if (pos >= end) {
return std::nullopt;
}
size_t value_end = pos;
if (content[value_end] == '-') {
++value_end;
}
while (value_end < end &&
std::isdigit(static_cast<unsigned char>(content[value_end]))) {
++value_end;
}
if (value_end == pos || (value_end == pos + 1 && content[pos] == '-')) {
return std::nullopt;
}
return std::stoi(content.substr(pos, value_end - pos));
}
ModelConfig LoadModelConfig(const std::string& model_path) {
const std::filesystem::path config_path =
std::filesystem::path(model_path) / "config.json";
std::ifstream ifs(config_path);
if (!ifs.is_open()) {
throw std::runtime_error("failed to open model config: " +
config_path.string());
}
std::stringstream buffer;
buffer << ifs.rdbuf();
const std::string content = RemoveJsonComments(buffer.str());
ModelConfig cfg;
cfg.hidden_size =
FindJsonIntValueInRange(content, 0, content.size(), "hidden_size")
.value_or(0);
cfg.vocab_size =
FindJsonIntValueInRange(content, 0, content.size(), "vocab_size")
.value_or(0);
cfg.max_position_embeddings =
FindJsonIntValueInRange(
content, 0, content.size(), "max_position_embeddings")
.value_or(0);
cfg.model_type =
FindJsonStringValueInRange(content, 0, content.size(), "model_type")
.value_or(FindJsonStringValueInRange(
content, 0, content.size(), "model_name")
.value_or(""));
size_t text_begin = 0;
size_t text_end = 0;
if (FindObjectRange(content, "text_config", &text_begin, &text_end) !=
nullptr) {
if (cfg.hidden_size <= 0) {
cfg.hidden_size =
FindJsonIntValueInRange(content, text_begin, text_end, "hidden_size")
.value_or(0);
}
if (cfg.vocab_size <= 0) {
cfg.vocab_size =
FindJsonIntValueInRange(content, text_begin, text_end, "vocab_size")
.value_or(0);
}
if (cfg.max_position_embeddings <= 0) {
cfg.max_position_embeddings =
FindJsonIntValueInRange(
content, text_begin, text_end, "max_position_embeddings")
.value_or(0);
}
}
if (cfg.hidden_size <= 0) {
throw std::runtime_error(
"config.json missing hidden_size/text_config.hidden_size");
}
if (cfg.vocab_size <= 0) {
throw std::runtime_error(
"config.json missing vocab_size/text_config.vocab_size");
}
if (cfg.max_position_embeddings <= 0) {
throw std::runtime_error(
"config.json missing "
"max_position_embeddings/text_config.max_position_embeddings");
}
return cfg;
}
std::string ResolveModelId(const std::string& model_path) {
std::filesystem::path path =
std::filesystem::path(model_path).lexically_normal();
if (path.has_filename()) {
return path.filename().string();
}
return path.string();
}
void PrintUsage(const char* argv0) {
std::cerr
<< "Usage: " << argv0 << " --model_path PATH [options]\n"
<< "Options:\n"
<< " --master_node_addr STR default: 127.0.0.1:18899\n"
<< " --prompt_size N number of requests kept in pool, default: "
"128\n"
<< " --token_min_size N minimum prompt token length, default: "
"1024\n"
<< " --token_max_size N maximum prompt token length, default: "
"1024\n"
<< " --qps FLOAT target QPS, default: 1.0\n"
<< " --client_threads N concurrent client request threads; 0 "
"means auto (ceil(qps))\n"
<< " --duration_s N duration in seconds, default: 60; 0 "
"means run forever\n"
<< " --mm_min_span N default: 8\n"
<< " --mm_max_span N default: 64\n"
<< " --seed N default: 20260410\n";
}
CliOptions ParseArgs(int argc, char** argv) {
CliOptions options;
for (int i = 1; i < argc; ++i) {
const std::string arg = argv[i];
auto next = [&](const char* name) -> std::string {
if (i + 1 >= argc) {
throw std::runtime_error(std::string("missing value for ") + name);
}
return argv[++i];
};
if (arg == "--model_path") {
options.model_path = next("--model_path");
} else if (arg == "--master_node_addr") {
options.master_node_addr = next("--master_node_addr");
} else if (arg == "--prompt_size") {
options.prompt_size = std::stoi(next("--prompt_size"));
} else if (arg == "--token_min_size") {
options.token_min_size = std::stoi(next("--token_min_size"));
} else if (arg == "--token_max_size") {
options.token_max_size = std::stoi(next("--token_max_size"));
} else if (arg == "--qps") {
options.qps = std::stod(next("--qps"));
} else if (arg == "--client_threads") {
options.client_threads = std::stoi(next("--client_threads"));
} else if (arg == "--duration_s") {
options.duration_s = std::stoi(next("--duration_s"));
} else if (arg == "--mm_min_span") {
options.mm_min_span = std::stoi(next("--mm_min_span"));
} else if (arg == "--mm_max_span") {
options.mm_max_span = std::stoi(next("--mm_max_span"));
} else if (arg == "--seed") {
options.seed = static_cast<uint32_t>(std::stoul(next("--seed")));
} else if (arg == "--help" || arg == "-h") {
PrintUsage(argv[0]);
std::exit(0);
} else {
throw std::runtime_error("unknown argument: " + arg);
}
}
if (options.model_path.empty()) {
throw std::runtime_error("--model_path is required");
}
if (options.prompt_size <= 0) {
throw std::runtime_error("--prompt_size must be > 0");
}
if (options.token_min_size <= 0 || options.token_max_size <= 0) {
throw std::runtime_error(
"--token_min_size and --token_max_size must be > 0");
}
if (options.token_min_size > options.token_max_size) {
throw std::runtime_error("--token_min_size must be <= --token_max_size");
}
if (options.qps <= 0.0) {
throw std::runtime_error("--qps must be > 0");
}
if (options.client_threads < 0) {
throw std::runtime_error("--client_threads must be >= 0");
}
if (options.duration_s < 0) {
throw std::runtime_error("--duration_s must be >= 0");
}
if (options.mm_min_span <= 0 || options.mm_max_span <= 0 ||
options.mm_min_span > options.mm_max_span) {
throw std::runtime_error("invalid mm span range");
}
return options;
}
std::vector<int32_t> MakeRandomTokens(int32_t token_size,
int32_t vocab_size,
std::mt19937& rng) {
std::uniform_int_distribution<int32_t> dist(1, std::max(2, vocab_size - 1));
std::vector<int32_t> token_ids(token_size);
for (int32_t& token_id : token_ids) {
token_id = dist(rng);
}
return token_ids;
}
std::vector<std::pair<uint32_t, uint32_t>> MakeRandomMmSpans(
int32_t token_size,
const CliOptions& options,
std::mt19937& rng) {
std::vector<std::pair<uint32_t, uint32_t>> spans;
spans.reserve(kFixedMmItemsPerRequest);
const int32_t max_valid_end = token_size - 1;
int32_t cursor = 1;
for (int32_t item_idx = 0; item_idx < kFixedMmItemsPerRequest; ++item_idx) {
const int32_t remaining_items = kFixedMmItemsPerRequest - item_idx;
const int32_t remaining_tokens = max_valid_end - cursor;
if (remaining_tokens <= options.mm_min_span) {
break;
}
const int32_t max_span =
std::min(options.mm_max_span, remaining_tokens - remaining_items + 1);
if (max_span < options.mm_min_span) {
break;
}
std::uniform_int_distribution<int32_t> span_dist(options.mm_min_span,
max_span);
const int32_t length = span_dist(rng);
const int32_t max_offset =
max_valid_end - length - (remaining_items - 1) * options.mm_min_span;
if (max_offset < cursor) {
break;
}
std::uniform_int_distribution<int32_t> offset_dist(cursor, max_offset);
const int32_t offset = offset_dist(rng);
spans.emplace_back(static_cast<uint32_t>(offset),
static_cast<uint32_t>(length));
cursor = offset + length + 1;
}
if (spans.empty()) {
const uint32_t fallback_len =
static_cast<uint32_t>(std::min(options.mm_min_span, token_size - 2));
spans.emplace_back(1U, std::max<uint32_t>(1U, fallback_len));
}
return spans;
}
std::vector<RequestPayload> BuildRequestPool(const CliOptions& options,
const ModelConfig& config) {
std::mt19937 rng(options.seed);
std::vector<RequestPayload> pool;
pool.reserve(options.prompt_size);
const int32_t max_model_token_size = config.max_position_embeddings - 1;
const int32_t token_min_size =
std::min(options.token_min_size, max_model_token_size);
const int32_t token_max_size =
std::min(options.token_max_size, max_model_token_size);
if (token_min_size <= 1 || token_max_size <= 1) {
throw std::runtime_error(
"token_size range is too large for model max_position_embeddings");
}
if (token_min_size > token_max_size) {
throw std::runtime_error(
"token_size range becomes invalid after clamping to model limits");
}
std::uniform_int_distribution<int32_t> token_size_dist(token_min_size,
token_max_size);
for (int32_t i = 0; i < options.prompt_size; ++i) {
const int32_t token_size = token_size_dist(rng);
RequestPayload payload;
payload.token_ids = MakeRandomTokens(token_size, config.vocab_size, rng);
pool.push_back(std::move(payload));
}
return pool;
}
void UpdateMax(std::atomic<uint64_t>& target, uint64_t value) {
uint64_t prev = target.load(std::memory_order_relaxed);
while (
prev < value &&
!target.compare_exchange_weak(
prev, value, std::memory_order_relaxed, std::memory_order_relaxed)) {
}
}
void RecordResponseMetrics(const XLLM_Response* resp,
uint64_t latency_us,
Metrics* metrics) {
metrics->sent.fetch_add(1, std::memory_order_relaxed);
metrics->total_latency_us.fetch_add(latency_us, std::memory_order_relaxed);
UpdateMax(metrics->max_latency_us, latency_us);
if (resp == nullptr) {
metrics->failed.fetch_add(1, std::memory_order_relaxed);
return;
}
metrics->total_prompt_tokens.fetch_add(
static_cast<uint64_t>(std::max(resp->usage.prompt_tokens, 0)),
std::memory_order_relaxed);
metrics->total_completion_tokens.fetch_add(
static_cast<uint64_t>(std::max(resp->usage.completion_tokens, 0)),
std::memory_order_relaxed);
if (resp->status_code == kSuccess) {
metrics->succeeded.fetch_add(1, std::memory_order_relaxed);
return;
}
metrics->failed.fetch_add(1, std::memory_order_relaxed);
if (resp->status_code == kTimeout) {
metrics->timeout.fetch_add(1, std::memory_order_relaxed);
} else if (resp->status_code == kInvalidRequest) {
metrics->invalid_request.fetch_add(1, std::memory_order_relaxed);
} else if (resp->status_code == kInternalError) {
metrics->internal_error.fetch_add(1, std::memory_order_relaxed);
}
}
void IncrementInFlight(Metrics* metrics) {
const uint64_t current =
metrics->current_in_flight.fetch_add(1, std::memory_order_relaxed) + 1;
UpdateMax(metrics->max_in_flight, current);
}
void DecrementInFlight(Metrics* metrics) {
metrics->current_in_flight.fetch_sub(1, std::memory_order_relaxed);
}
void PrintSummary(const CliOptions& options,
const ModelConfig& config,
const Metrics& metrics,
double actual_duration_s) {
const uint64_t sent = metrics.sent.load(std::memory_order_relaxed);
const uint64_t succeeded = metrics.succeeded.load(std::memory_order_relaxed);
const uint64_t failed = metrics.failed.load(std::memory_order_relaxed);
const uint64_t total_latency_us =
metrics.total_latency_us.load(std::memory_order_relaxed);
const uint64_t max_latency_us =
metrics.max_latency_us.load(std::memory_order_relaxed);
const uint64_t max_in_flight =
metrics.max_in_flight.load(std::memory_order_relaxed);
const double avg_latency_ms = sent == 0
? 0.0
: static_cast<double>(total_latency_us) /
static_cast<double>(sent) / 1000.0;
const double actual_qps = actual_duration_s <= 0.0
? 0.0
: static_cast<double>(sent) / actual_duration_s;
std::cout << "=== stress_rec_multimodal_completions summary ===\n";
std::cout << "model_id=" << ResolveModelId(options.model_path) << "\n";
std::cout << "model_type=" << config.model_type << "\n";
std::cout << "devices=" << options.devices << "\n";
std::cout << "master_node_addr=" << options.master_node_addr << "\n";
std::cout << "hidden_size=" << config.hidden_size
<< ", vocab_size=" << config.vocab_size
<< ", max_position_embeddings=" << config.max_position_embeddings
<< "\n";
std::cout << "prompt_size=" << options.prompt_size << ", token_size_range=["
<< options.token_min_size << ", " << options.token_max_size << "]"
<< ", qps_target=" << options.qps
<< ", duration_s=" << options.duration_s << "\n";
std::cout << "client_threads="
<< (options.client_threads == 0
? static_cast<int32_t>(std::ceil(options.qps))
: options.client_threads)
<< "\n";
std::cout << "mm_span_range=[" << options.mm_min_span << ", "
<< options.mm_max_span
<< "], mm_items_per_request=" << kFixedMmItemsPerRequest << "\n";
std::cout << "sent=" << sent << ", succeeded=" << succeeded
<< ", failed=" << failed << "\n";
std::cout << "max_client_in_flight=" << max_in_flight << "\n";
std::cout << "timeout=" << metrics.timeout.load(std::memory_order_relaxed)
<< ", invalid_request="
<< metrics.invalid_request.load(std::memory_order_relaxed)
<< ", internal_error="
<< metrics.internal_error.load(std::memory_order_relaxed) << "\n";
std::cout << std::fixed << std::setprecision(3)
<< "actual_duration_s=" << actual_duration_s
<< ", actual_qps=" << actual_qps
<< ", avg_latency_ms=" << avg_latency_ms << ", max_latency_ms="
<< static_cast<double>(max_latency_us) / 1000.0 << "\n";
std::cout << "prompt_tokens_total="
<< metrics.total_prompt_tokens.load(std::memory_order_relaxed)
<< ", completion_tokens_total="
<< metrics.total_completion_tokens.load(std::memory_order_relaxed)
<< "\n";
}
} // namespace
int main(int argc, char** argv) {
try {
const CliOptions options = ParseArgs(argc, argv);
const ModelConfig config = LoadModelConfig(options.model_path);
auto request_pool = BuildRequestPool(options, config);
const std::string model_id = ResolveModelId(options.model_path);
std::cout << "Loaded model config from " << options.model_path << "\n";
std::cout << "hidden_size=" << config.hidden_size
<< ", vocab_size=" << config.vocab_size
<< ", max_position_embeddings=" << config.max_position_embeddings
<< ", model_type=" << config.model_type << "\n";
std::cout << "model_id=" << model_id << "\n";
XLLM_REC_Handler* rec_handler = xllm_rec_create();
if (rec_handler == nullptr) {
throw std::runtime_error("xllm_rec_create returned nullptr");
}
XLLM_InitOptions init_options;
xllm_rec_init_options_default(&init_options);
std::snprintf(init_options.master_node_addr,
sizeof(init_options.master_node_addr),
"%s",
options.master_node_addr.c_str());
const bool init_ok = xllm_rec_initialize(rec_handler,
options.model_path.c_str(),
options.devices.c_str(),
&init_options);
if (!init_ok) {
xllm_rec_destroy(rec_handler);
throw std::runtime_error("xllm_rec_initialize failed");
}
XLLM_RequestParams request_params;
xllm_rec_request_params_default(&request_params);
request_params.max_tokens = kFixedRequestMaxTokens;
request_params.beam_width = kFixedRequestBeamWidth;
request_params.logprobs = kFixedRequestLogprobs;
request_params.top_k = kFixedRequestTopK;
request_params.top_logprobs = kFixedRequestTopLogprobs;
std::cout << "Initialized REC with fixed request params: max_tokens="
<< request_params.max_tokens
<< ", beam_width=" << request_params.beam_width
<< ", logprobs=" << (request_params.logprobs ? 1 : 0)
<< ", top_k=" << request_params.top_k
<< ", top_logprobs=" << request_params.top_logprobs << "\n";
Metrics metrics;
const auto start_time = Clock::now();
const bool run_forever = options.duration_s == 0;
const auto stop_time =
run_forever ? Clock::time_point::max()
: start_time + std::chrono::seconds(options.duration_s);
const double interval_us = 1e6 / options.qps;
std::mutex queue_mutex;
std::condition_variable queue_cv;
std::deque<ScheduledRequest> queue;
bool producer_done = false;
const int32_t auto_worker_count =
std::max<int32_t>(1, static_cast<int32_t>(std::ceil(options.qps)));
const int32_t worker_count = std::max<int32_t>(
1,
std::min<int32_t>(kMaxClientParallelism,
options.client_threads > 0 ? options.client_threads
: auto_worker_count));
std::cout << "Using client_threads=" << worker_count
<< (options.client_threads > 0 ? " (explicit)" : " (auto)")
<< "\n";
auto worker_fn = [&](int32_t worker_id) {
std::mt19937 rng(options.seed ^ 0x9e3779b9U ^
static_cast<uint32_t>(worker_id * 0x85ebca6bU));
EmbeddingMmDataBuilder mm_builder;
while (true) {
ScheduledRequest scheduled;
{
std::unique_lock<std::mutex> lock(queue_mutex);
queue_cv.wait(lock,
[&]() { return producer_done || !queue.empty(); });
if (queue.empty()) {
if (producer_done) {
return;
}
continue;
}
scheduled = queue.front();
queue.pop_front();
}
const RequestPayload& payload = request_pool[scheduled.pool_index];
const auto mm_spans = MakeRandomMmSpans(
static_cast<int32_t>(payload.token_ids.size()), options, rng);
const XLLM_MM_Data* mm_data = mm_builder.Build(
mm_spans, config.hidden_size, scheduled.request_index, rng);
IncrementInFlight(&metrics);
const auto req_begin = Clock::now();
XLLM_Response* resp =
xllm_rec_multimodal_completions(rec_handler,
model_id.c_str(),
payload.token_ids.data(),
payload.token_ids.size(),
mm_data,
options.timeout_ms,
&request_params);
const auto req_end = Clock::now();
DecrementInFlight(&metrics);
const uint64_t latency_us = static_cast<uint64_t>(
std::chrono::duration_cast<std::chrono::microseconds>(req_end -
req_begin)
.count());
RecordResponseMetrics(resp, latency_us, &metrics);
if (resp != nullptr && resp->status_code != kSuccess) {
std::cerr << "request " << scheduled.request_index
<< " failed: status=" << resp->status_code
<< ", error=" << resp->error_info << "\n";
std::cerr << "request " << scheduled.request_index
<< " shape: token_size=" << payload.token_ids.size()
<< ", mm_items=" << mm_spans.size();
for (const auto& [offset, length] : mm_spans) {
std::cerr << " [" << offset << "," << length << "]";
}
std::cerr << "\n";
}
xllm_rec_free_response(resp);
}
};
std::vector<std::thread> workers;
workers.reserve(static_cast<size_t>(worker_count));
for (int32_t worker_id = 0; worker_id < worker_count; ++worker_id) {
workers.emplace_back(worker_fn, worker_id);
}
uint64_t request_index = 0;
size_t next_pool_index = 0;
while (Clock::now() < stop_time) {
const auto scheduled_time =
start_time + std::chrono::microseconds(
static_cast<int64_t>(request_index * interval_us));
std::this_thread::sleep_until(scheduled_time);
{
std::lock_guard<std::mutex> lock(queue_mutex);
queue.push_back(ScheduledRequest{request_index, next_pool_index});
}
queue_cv.notify_one();
next_pool_index = (next_pool_index + 1) % request_pool.size();
++request_index;
}
{
std::lock_guard<std::mutex> lock(queue_mutex);
producer_done = true;
}
queue_cv.notify_all();
for (auto& worker : workers) {
worker.join();
}
const double actual_duration_s =
std::chrono::duration_cast<std::chrono::duration<double>>(Clock::now() -
start_time)
.count();
PrintSummary(options, config, metrics, actual_duration_s);
xllm_rec_destroy(rec_handler);
return 0;
} catch (const std::exception& e) {
std::cerr << "fatal: " << e.what() << "\n";
return 1;
}
}

113
upstream_ref/xllm/xllm/c_api/examples/test_query_llm_chat_completions.cpp Normal file
View File

@@ -0,0 +1,113 @@
/* Copyright 2025 The xLLM Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
https://github.com/jd-opensource/xllm/blob/main/LICENSE
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#include <unistd.h>
#include <cstring>
#include <iostream>
#include "llm.h"
std::string devices = "cuda:4";
std::string model_name = "Qwen3-8B";
std::string model_path = "/export/home/models/Qwen3-8B";
XLLM_LLM_Handler* service_startup_hook() {
XLLM_LLM_Handler* llm_handler = xllm_llm_create();
// If there is no separate setting, init_options can be passed as nullptr, and
// the default value(XLLM_INIT_LLM_OPTIONS_DEFAULT) will be used
XLLM_InitLLMOptions init_options;
xllm_llm_init_options_default(&init_options);
snprintf(
init_options.log_dir, sizeof(init_options.log_dir), "/export/xllm/log");
bool ret = xllm_llm_initialize(
llm_handler, model_path.c_str(), devices.c_str(), &init_options);
if (!ret) {
std::cout << "LLM init failed" << std::endl;
xllm_llm_destroy(llm_handler);
return nullptr;
}
std::cout << "LLM init successfully" << std::endl;
return llm_handler;
}
void service_stop_hook(XLLM_LLM_Handler* llm_handler) {
xllm_llm_destroy(llm_handler);
std::cout << "LLM stop" << std::endl;
}
int main(int argc, char** argv) {
XLLM_LLM_Handler* llm_handler = service_startup_hook();
if (nullptr == llm_handler) {
return -1;
}
// If there is no separate setting, request_params can be passed as nullptr,
// and the default value(XLLM_REQUEST_PARAMS_DEFAULT) will be used
XLLM_RequestParams request_params;
xllm_llm_request_params_default(&request_params);
request_params.max_tokens = 300;
std::string content =
"You are an expert in e-commerce scenarios. The current scenario is an "
"e-commerce search engine with a comprehensive range of business "
"categories. Your task is to determine whether 'user query' and "
"'product title' are related in the e-commerce search engine. "
"Discrimination criteria: If the search for 'user query' returns' "
"product title 'that meets the user's needs, then the task is "
"relevant. Output requirement: Please provide the answer in the "
"'related' or 'unrelated' section, without mentioning any other "
"content. User query: 'Hotpot sauce'. Product title: 'Grassland Red "
"Sun Hotpot Base Dip Multi flavored Barbecue Sauce Tomato Sauce Leek "
"Flower Sauce Nightsnack Paired with [New] Spicy Barbecue Sauce 100g'";
XLLM_ChatMessage message = {0};
strncpy(message.role, "user", sizeof(message.role) - 1);
message.content = const_cast<char*>(content.c_str());
XLLM_Response* resp = xllm_llm_chat_completions(
llm_handler, model_name.c_str(), &message, 1, 10000, &request_params);
if (nullptr == resp) {
std::cout << "LLM completions failed, response is nullptr" << std::endl;
service_stop_hook(llm_handler);
return -1;
}
if (resp->status_code != XLLM_StatusCode::kSuccess) {
std::cout << "LLM completions failed, status code:" << resp->status_code
<< ", error info:" << resp->error_info << std::endl;
} else {
std::cout << "LLM completions successfully" << std::endl;
if (nullptr != resp->choices.entries) {
for (int i = 0; i < resp->choices.entries_size; ++i) {
XLLM_Choice& choice = resp->choices.entries[i];
std::cout << "xllm answer[" << choice.index
<< "]:" << "role:" << choice.message->role
<< ",content:" << choice.message->content << std::endl;
}
}
}
xllm_llm_free_response(resp);
service_stop_hook(llm_handler);
return 0;
}

97
upstream_ref/xllm/xllm/c_api/examples/test_query_llm_completions.cpp Normal file
View File

@@ -0,0 +1,97 @@
/* Copyright 2025 The xLLM Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
https://github.com/jd-opensource/xllm/blob/main/LICENSE
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#include <unistd.h>
#include <cstring>
#include <iostream>
#include "llm.h"
std::string devices = "cuda:1";
std::string model_name = "Qwen3-8B";
std::string model_path = "/export/home/models/Qwen3-8B";
XLLM_LLM_Handler* service_startup_hook() {
XLLM_LLM_Handler* llm_handler = xllm_llm_create();
// If there is no separate setting, init_options can be passed as nullptr, and
// the default value(XLLM_INIT_LLM_OPTIONS_DEFAULT) will be used
XLLM_InitOptions init_options;
xllm_llm_init_options_default(&init_options);
snprintf(
init_options.log_dir, sizeof(init_options.log_dir), "/export/xllm/log");
bool ret = xllm_llm_initialize(
llm_handler, model_path.c_str(), devices.c_str(), &init_options);
if (!ret) {
std::cout << "LLM init failed" << std::endl;
xllm_llm_destroy(llm_handler);
return nullptr;
}
std::cout << "LLM init successfully" << std::endl;
return llm_handler;
}
void service_stop_hook(XLLM_LLM_Handler* llm_handler) {
xllm_llm_destroy(llm_handler);
std::cout << "LLM stop" << std::endl;
}
int main(int argc, char** argv) {
XLLM_LLM_Handler* llm_handler = service_startup_hook();
if (nullptr == llm_handler) {
return -1;
}
// If there is no separate setting, request_params can be passed as nullptr,
// and the default value(XLLM_REQUEST_PARAMS_DEFAULT) will be used
XLLM_RequestParams request_params;
xllm_llm_request_params_default(&request_params);
request_params.max_tokens = 300;
std::string prompt = "please briefly introduce XLLM for me";
XLLM_Response* resp = xllm_llm_completions(
llm_handler, model_name.c_str(), prompt.c_str(), 10000, &request_params);
if (nullptr == resp) {
std::cout << "LLM completions failed, response is nullptr" << std::endl;
service_stop_hook(llm_handler);
return -1;
}
if (resp->status_code != XLLM_StatusCode::kSuccess) {
std::cout << "LLM completions failed, status code:" << resp->status_code
<< ", error info:" << resp->error_info << std::endl;
} else {
std::cout << "LLM completions successfully" << std::endl;
if (nullptr != resp->choices.entries) {
for (int i = 0; i < resp->choices.entries_size; ++i) {
XLLM_Choice& choice = resp->choices.entries[i];
std::cout << "xllm answer[" << choice.index << "]:" << choice.text
<< std::endl;
}
}
}
xllm_llm_free_response(resp);
service_stop_hook(llm_handler);
return 0;
}

153
upstream_ref/xllm/xllm/c_api/examples/test_query_rec_completions.cpp Normal file
View File

@@ -0,0 +1,153 @@
/* Copyright 2025 The xLLM Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
https://github.com/jd-opensource/xllm/blob/main/LICENSE
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#include <unistd.h>
#include <cstring>
#include <iostream>
#include <random>
#include "rec.h"
#if defined(USE_NPU)
std::string devices = "npu:14";
#elif defined(USE_CUDA)
std::string devices = "cuda:0";
#else
std::string devices = "npu:14";
#endif
std::string model_name = "Qwen3-0.6B";
std::string model_path = "/export/home/models/Qwen3-0.6B";
XLLM_REC_Handler* service_startup_hook() {
XLLM_REC_Handler* rec_handler = xllm_rec_create();
// If there is no separate setting, init_options can be passed as nullptr, and
// the default value(XLLM_INIT_REC_OPTIONS_DEFAULT) will be used
XLLM_InitOptions init_options;
xllm_rec_init_options_default(&init_options);
init_options.block_size = 1;
init_options.max_tokens_per_batch = 8192;
init_options.max_seqs_per_batch = 4;
init_options.max_memory_utilization = 0.8;
init_options.max_cache_size = 500000;
init_options.beam_width = 64;
init_options.max_decode_rounds = 3;
init_options.enable_chunked_prefill = false;
init_options.enable_prefix_cache = false;
#if defined(USE_NPU)
init_options.enable_graph = false;
init_options.enable_graph_mode_decode_no_padding = false;
init_options.enable_prefill_piecewise_graph = false;
init_options.rec_worker_max_concurrency = 1;
#endif
bool ret = xllm_rec_initialize(
rec_handler, model_path.c_str(), devices.c_str(), &init_options);
if (!ret) {
std::cout << "REC init failed" << std::endl;
xllm_rec_destroy(rec_handler);
return nullptr;
}
std::cout << "REC init successfully" << std::endl;
return rec_handler;
}
void service_stop_hook(XLLM_REC_Handler* rec_handler) {
xllm_rec_destroy(rec_handler);
std::cout << "REC stop" << std::endl;
}
int main(int argc, char** argv) {
if (argc > 1) {
devices = argv[1];
}
std::cout << "Using model path: " << model_path << std::endl;
std::cout << "Using devices: " << devices << std::endl;
XLLM_REC_Handler* rec_handler = service_startup_hook();
if (nullptr == rec_handler) {
return -1;
}
// If there is no separate setting, request_params can be passed as nullptr,
// and the default value(XLLM_REQUEST_PARAMS_DEFAULT) will be used
XLLM_RequestParams request_params;
xllm_rec_request_params_default(&request_params);
request_params.max_tokens = 3;
request_params.beam_width = 64;
request_params.logprobs = true;
// request_params.temperature = 1.0;
request_params.top_k = 64;
request_params.top_logprobs = 64;
// request_params.top_p = 1.0;
// request_params.repetition_penalty = 1.0;
// Qwen3-0.6B tokenizer ids for: "where is bejing?".
std::vector<int32_t> token_ids = {2870, 374, 387, 98168, 30};
size_t token_size = token_ids.size();
const int32_t* token_ids_ptr = token_ids.data();
XLLM_Response* resp = xllm_rec_token_completions(rec_handler,
model_name.c_str(),
token_ids_ptr,
token_size,
100000,
&request_params);
if (nullptr == resp) {
std::cout << "REC completions failed, response is nullptr" << std::endl;
service_stop_hook(rec_handler);
return -1;
}
if (resp->status_code != XLLM_StatusCode::kSuccess) {
std::cout << "REC completions failed, status code:" << resp->status_code
<< ", error info:" << resp->error_info << std::endl;
} else {
std::cout << "REC completions successfully, size:"
<< resp->choices.entries_size << std::endl;
if (nullptr != resp->choices.entries) {
for (int i = 0; i < resp->choices.entries_size; ++i) {
XLLM_Choice& choice = resp->choices.entries[i];
std::cout << "token size: " << choice.token_size
<< ",logprobs size:" << choice.logprobs.entries_size
<< std::endl;
for (int j = 0; j < choice.token_size; j++) {
std::cout << "xllm answer[" << choice.index
<< "]: token id=" << choice.token_ids[j] << std::endl;
}
for (int j = 0; j < choice.logprobs.entries_size; j++) {
XLLM_LogProb& logprob = choice.logprobs.entries[j];
std::cout << "xllm answer[" << choice.index
<< "]: token id=" << logprob.token_id
<< ", token logprob=" << logprob.logprob << std::endl;
}
}
}
}
xllm_rec_free_response(resp);
service_stop_hook(rec_handler);
return 0;
}

376
upstream_ref/xllm/xllm/c_api/examples/test_query_rec_multimodal_completions.cpp Normal file
View File

@@ -0,0 +1,376 @@
/* Copyright 2026 The xLLM Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
https://github.com/jd-opensource/xllm/blob/main/LICENSE
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#include <unistd.h>
#include <cstdint>
#include <cstdio>
#include <cstring>
#include <iostream>
#include <memory>
#include <random>
#include <stdexcept>
#include <utility>
#include <vector>
#include "rec.h"
std::string devices = "cuda:0";
std::string model_name = "homepage_qwen_06b_6_raw";
std::string model_path = "/export/home/models/homepage_qwen_06b_6_raw";
#define MODEL_WORD_EMBEDDING_SIZE 1024
class XLLM_MM_Data_Wrapper {
public:
XLLM_MM_Data_Wrapper() = default;
~XLLM_MM_Data_Wrapper() { reset(); }
XLLM_MM_Data_Wrapper(const XLLM_MM_Data_Wrapper&) = delete;
XLLM_MM_Data_Wrapper& operator=(const XLLM_MM_Data_Wrapper&) = delete;
bool build(
const std::vector<std::pair<uint32_t, uint32_t>>& token_positions) {
if (is_built_ || token_positions.empty()) {
fprintf(stderr,
"build() failed: already built or empty token positions\n");
return false;
}
mm_data_.type_mask = static_cast<uint32_t>(XLLM_MM_TYPE_EMBEDDING);
mm_data_.is_dict = false;
for (size_t i = 0; i < token_positions.size(); ++i) {
const auto& [offset, length] = token_positions[i];
if (length == 0) {
fprintf(stderr, "build() skipped item %zu: length cannot be 0\n", i);
continue;
}
items_.emplace_back(create_embedding_item(offset, length));
}
if (items_.empty()) {
fprintf(stderr, "build() failed: no valid embedding items created\n");
return false;
}
mm_data_.data.items.entries_size = items_.size();
mm_data_.data.items.entries = items_.data();
is_built_ = true;
return true;
}
void reset() {
memset(&mm_data_, 0, sizeof(mm_data_));
items_.clear();
tensor_buffers_.clear();
is_built_ = false;
}
const XLLM_MM_Data* get_data() const {
return is_built_ ? &mm_data_ : nullptr;
}
void validate() const {
if (!is_built_) {
fprintf(stderr,
"validate() failed: no data available (call build() first)\n");
return;
}
const size_t item_count = mm_data_.data.items.entries_size;
printf("=== Validating %zu Embedding Items ===\n\n", item_count);
for (size_t i = 0; i < item_count; ++i) {
const auto& item = mm_data_.data.items.entries[i];
printf("=== Embedding Item %zu ===\n", i + 1);
printf("Token Position: offset=%u, length=%u\n",
item.state.token_pos.offset,
item.state.token_pos.length);
printf("Data Type: (%d)\n", item.data.data.tensor.dtype);
printf("Tensor Shape: rank=%d, dim=[%d, %d]\n\n",
item.data.data.tensor.dims.rank,
item.data.data.tensor.dims.dim[0],
item.data.data.tensor.dims.dim[1]);
}
}
bool is_built() const { return is_built_; }
size_t get_item_count() const {
return is_built_ ? mm_data_.data.items.entries_size : 0;
}
private:
XLLM_MM_Data mm_data_{};
std::vector<XLLM_MM_Item> items_;
std::vector<std::unique_ptr<uint16_t[]>> tensor_buffers_;
bool is_built_ = false;
inline uint16_t float_to_bfloat16(float f) {
union {
float f32;
uint32_t u32;
} u;
u.f32 = f;
return static_cast<uint16_t>(u.u32 >> 16);
}
XLLM_MM_Item create_embedding_item(uint32_t offset, uint32_t length) {
XLLM_MM_Item item{};
item.type = XLLM_MM_TYPE_EMBEDDING;
item.state.token_pos.offset = offset;
item.state.token_pos.length = length;
item.data.is_single_tensor = true;
item.data.data.tensor.dtype = XLLM_DTYPE_BFLOAT16;
item.data.data.tensor.dims.rank = 2;
memset(item.data.data.tensor.dims.dim,
0,
sizeof(item.data.data.tensor.dims.dim));
item.data.data.tensor.dims.dim[0] = static_cast<int>(length);
item.data.data.tensor.dims.dim[1] = MODEL_WORD_EMBEDDING_SIZE;
const size_t element_count = length * MODEL_WORD_EMBEDDING_SIZE;
const size_t buffer_size_bytes = element_count * sizeof(uint16_t);
auto buffer = std::make_unique<uint16_t[]>(element_count);
for (size_t i = 0; i < length; ++i) {
for (size_t j = 0; j < MODEL_WORD_EMBEDDING_SIZE; ++j) {
float float_val =
static_cast<float>(i * MODEL_WORD_EMBEDDING_SIZE + j) /
static_cast<float>(element_count);
uint16_t bf16_val = float_to_bfloat16(float_val);
buffer[i * MODEL_WORD_EMBEDDING_SIZE + j] = bf16_val;
}
}
item.data.data.tensor.data = buffer.get();
tensor_buffers_.push_back(std::move(buffer));
return item;
}
};
XLLM_REC_Handler* service_startup_hook() {
XLLM_REC_Handler* rec_handler = xllm_rec_create();
// If there is no separate setting, init_options can be passed as nullptr, and
// the default value(XLLM_INIT_REC_OPTIONS_DEFAULT) will be used
XLLM_InitOptions init_options;
xllm_rec_init_options_default(&init_options);
// init_options.beam_width = 1;
// init_options.max_decode_rounds = 0;
snprintf(init_options.log_dir,
sizeof(init_options.log_dir),
"/export/home/huheng7/log");
bool ret = xllm_rec_initialize(
rec_handler, model_path.c_str(), devices.c_str(), &init_options);
if (!ret) {
std::cout << "REC init failed" << std::endl;
xllm_rec_destroy(rec_handler);
return nullptr;
}
std::cout << "REC init successfully" << std::endl;
return rec_handler;
}
void service_stop_hook(XLLM_REC_Handler* rec_handler) {
xllm_rec_destroy(rec_handler);
std::cout << "REC stop" << std::endl;
}
int generate_random_int(int min, int max) {
if (min > max) {
throw std::invalid_argument("min cannot be greater than max");
}
static std::random_device rd;
static std::mt19937 gen(rd());
std::uniform_int_distribution<int> dist(min, max);
return dist(gen);
}
int main(int argc, char** argv) {
XLLM_REC_Handler* rec_handler = service_startup_hook();
if (nullptr == rec_handler) {
return -1;
}
// If there is no separate setting, request_params can be passed as nullptr,
// and the default value(XLLM_REQUEST_PARAMS_DEFAULT) will be used
XLLM_RequestParams request_params;
xllm_rec_request_params_default(&request_params);
// request_params.beam_width = 128;
request_params.max_tokens = 3;
request_params.beam_width = 128;
request_params.logprobs = true;
// request_params.temperature = 1.0;
request_params.top_k = 128;
request_params.top_logprobs = 128;
// request_params.top_p = 1.0;
// request_params.repetition_penalty = 1.0;
std::vector<int32_t> token_ids = {
151644, 8948, 198, 56568, 101909, 101215, 104799, 101914, 101057,
3837, 103929, 100032, 44956, 15946, 55338, 45943, 104570, 11622,
105801, 72881, 64559, 307, 71817, 51463, 3837, 56568, 107618,
100345, 20002, 104754, 72651, 105565, 45943, 116951, 101034, 67949,
72651, 109348, 36407, 104538, 20002, 104326, 87267, 72651, 109348,
1773, 151645, 198, 151644, 872, 198, 20002, 21, 15,
35727, 31843, 36667, 59879, 20450, 99805, 32044, 72651, 105565,
45943, 32044, 113507, 153479, 155828, 160439, 11, 153479, 157177,
160439, 11, 153479, 155828, 160439, 11, 153479, 155828, 160439,
11, 153479, 155828, 160439, 11, 153479, 155828, 160439, 11,
155622, 158228, 160337, 11, 152907, 158228, 159858, 11, 153036,
158228, 160333, 11, 153258, 159797, 160105, 11, 153186, 157627,
160740, 11, 152907, 158228, 160680, 11, 154562, 157329, 160321,
11, 153326, 157680, 163928, 11, 153258, 159634, 160105, 11,
152847, 157129, 162841, 11, 152847, 157399, 162841, 11, 152847,
158228, 163388, 11, 153036, 159807, 162840, 11, 154562, 157329,
160321, 11, 154562, 156839, 160321, 11, 154562, 158181, 160321,
11, 153326, 158534, 163886, 11, 153326, 157177, 163041, 11,
155622, 158228, 163359, 11, 152569, 155800, 162738, 11, 153390,
158228, 160357, 11, 152663, 157649, 162738, 11, 155193, 158667,
162738, 11, 155622, 158228, 160706, 11, 151685, 158473, 162738,
11, 152907, 158228, 162653, 11, 151876, 158228, 159909, 11,
152907, 158228, 162407, 11, 152907, 158228, 163551, 11, 151685,
158473, 162738, 11, 152686, 155927, 162029, 11, 152663, 158228,
161841, 11, 152686, 155927, 162603, 11, 153516, 157280, 161980,
11, 153516, 159807, 160708, 11, 153516, 157900, 163856, 11,
153516, 155967, 161020, 11, 153516, 157280, 160838, 11, 153200,
157591, 162582, 11, 151924, 158696, 160358, 11, 154562, 159113,
160860, 11, 153386, 159086, 161519, 11, 154625, 159807, 160781,
11, 153479, 155828, 160439, 11, 153479, 155828, 160439, 11,
153479, 157177, 160439, 11, 153479, 155828, 160439, 11, 154213,
157866, 160523, 11, 153036, 156918, 163610, 11, 153036, 157351,
160974, 11, 153688, 158228, 160337, 11, 155507, 159807, 162736,
11, 155370, 159219, 161059, 11, 155002, 158118, 160019, 11,
155370, 159219, 161059, 11, 153792, 159022, 161003, 11, 155576,
155927, 161581, 11, 155576, 155927, 163189, 11, 155576, 159630,
162853, 11, 155576, 159630, 163527, 11, 155576, 159630, 162164,
11, 155576, 158048, 163339, 11, 155576, 157177, 163339, 11,
155576, 159630, 163527, 11, 155576, 157177, 163339, 11, 155576,
157680, 163339, 11, 155576, 159630, 160653, 11, 155576, 159630,
162153, 11, 155576, 159630, 161747, 11, 155576, 157505, 163339,
11, 153831, 158228, 160026, 11, 153390, 158228, 161841, 11,
153831, 156324, 162738, 11, 153390, 158228, 161491, 11, 153390,
159145, 162738, 11, 155507, 158473, 162738, 11, 153831, 157649,
162738, 11, 155507, 157770, 162738, 11, 153390, 158228, 161033,
11, 155507, 158473, 162738, 11, 153390, 158228, 160824, 11,
153479, 157649, 160439, 11, 153479, 157649, 160439, 11, 153479,
155828, 160439, 11, 153479, 157649, 160439, 11, 153479, 157649,
160439, 11, 153479, 157649, 160439, 11, 153849, 159380, 162841,
11, 152663, 158107, 162738, 11, 152271, 157371, 161110, 11,
152663, 157176, 160199, 11, 154936, 158966, 162841, 11, 153390,
158228, 161491, 11, 153036, 158228, 162840, 11, 155646, 158228,
162408, 11, 152663, 156814, 162738, 11, 152569, 158473, 162738,
11, 155646, 158228, 161308, 11, 152663, 158228, 163631, 11,
155370, 159786, 163029, 11, 153534, 159283, 161094, 11, 153534,
157756, 163778, 11, 151905, 156698, 163573, 11, 151905, 156698,
161534, 11, 151905, 156698, 162140, 11, 153534, 157931, 161817,
11, 153534, 157121, 161059, 11, 154826, 158585, 163433, 11,
154826, 158585, 160756, 11, 154826, 157666, 161504, 11, 154826,
157351, 161808, 11, 154826, 158585, 161062, 11, 154826, 157666,
161504, 11, 154826, 156537, 163635, 11, 155370, 159219, 161059,
11, 155370, 156903, 160381, 11, 155370, 156903, 160381, 11,
155370, 159219, 162223, 11, 155370, 159330, 162223, 11, 153464,
159219, 161059, 11, 154809, 156903, 160381, 11, 153464, 156878,
162223, 11, 154809, 157794, 162010, 11, 154809, 159219, 161059,
11, 151893, 159807, 162666, 11, 151893, 158534, 160890, 11,
153326, 157177, 163620, 11, 153326, 159462, 163041, 11, 152663,
156348, 162738, 11, 152663, 158473, 162736, 11, 152463, 156537,
160873, 11, 155507, 157176, 162738, 11, 155193, 158473, 162738,
11, 152663, 157649, 162738, 11, 152663, 158107, 162738, 11,
152663, 155780, 162738, 11, 152663, 158473, 162738, 11, 152663,
157649, 162738, 11, 152663, 157649, 162738, 11, 152663, 155828,
162738, 11, 152663, 158621, 162738, 11, 152663, 157176, 162738,
11, 155646, 158228, 160017, 11, 155682, 158228, 162859, 67949,
103969, 72651, 109348, 17714, 155646, 158228, 162234, 1773, 104210,
67949, 9370, 72651, 45943, 9370, 111450, 37945, 104538, 20002,
104326, 104309, 72651, 9370, 16, 15, 18947, 45943, 3837,
11622, 107463, 17992, 71817, 17177, 99859, 1773, 151645, 198,
151644, 77091, 198};
size_t token_size = token_ids.size();
const int32_t* token_ids_ptr = token_ids.data();
XLLM_MM_Data_Wrapper multimodal_data_wrapper;
std::vector<std::pair<uint32_t, uint32_t>> positions = {{100, 32}, {300, 64}};
multimodal_data_wrapper.build(positions);
multimodal_data_wrapper.validate();
// multimodal_data_wrapper.get_data(),
XLLM_Response* resp =
xllm_rec_multimodal_completions(rec_handler,
model_name.c_str(),
token_ids_ptr,
token_size,
multimodal_data_wrapper.get_data(),
10000,
&request_params);
if (nullptr == resp) {
std::cout << "REC completions failed, response is nullptr" << std::endl;
service_stop_hook(rec_handler);
return -1;
}
if (resp->status_code != XLLM_StatusCode::kSuccess) {
std::cout << "REC completions failed, status code:" << resp->status_code
<< ", error info:" << resp->error_info << std::endl;
} else {
std::cout << "REC completions successfully, size:"
<< resp->choices.entries_size << std::endl;
if (nullptr != resp->choices.entries) {
for (int i = 0; i < resp->choices.entries_size; ++i) {
XLLM_Choice& choice = resp->choices.entries[i];
std::cout << "token size: " << choice.token_size
<< ",logprobs size:" << choice.logprobs.entries_size
<< std::endl;
for (int j = 0; j < choice.token_size; j++) {
std::cout << "xllm answer[" << choice.index
<< "]: token id=" << choice.token_ids[j] << std::endl;
}
for (int j = 0; j < choice.logprobs.entries_size; j++) {
XLLM_LogProb& logprob = choice.logprobs.entries[j];
std::cout << "xllm answer[" << choice.index
<< "]: token id=" << logprob.token_id
<< ", token logprob=" << logprob.logprob << std::endl;
}
}
}
}
xllm_rec_free_response(resp);
service_stop_hook(rec_handler);
return 0;
}

751
upstream_ref/xllm/xllm/c_api/internal/helper.cpp Normal file
View File

@@ -0,0 +1,751 @@
/* Copyright 2025 The xLLM Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
https://github.com/jd-opensource/xllm/blob/main/LICENSE
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#include "helper.h"
#include <glog/logging.h>
#include <pthread.h>
#include <torch/torch.h>
#include <atomic>
#include <string>
#include "core/common/global_flags.h"
#include "core/util/env_var.h"
#include "core/util/rec_model_utils.h"
#include "core/util/uuid.h"
namespace xllm {
namespace helper {
namespace {
thread_local ShortUUID short_uuid;
static std::atomic<bool> g_glog_inited = false;
static pthread_mutex_t g_log_init_mutex = PTHREAD_MUTEX_INITIALIZER;
} // namespace
std::string generate_request_id() {
return "xllm-" + InstanceName::name()->get_name_hash() + "-" +
short_uuid.random();
}
void init_log(const std::string& log_dir) {
if (g_glog_inited.load(std::memory_order_acquire)) {
return;
}
pthread_mutex_lock(&g_log_init_mutex);
if (!g_glog_inited.load(std::memory_order_relaxed)) {
google::InitGoogleLogging("xllm");
std::string log_prefix = log_dir.empty() ? "./" : log_dir + "/";
google::SetLogDestination(google::INFO,
(log_prefix + "xllm.log.INFO.").c_str());
google::SetLogDestination(google::WARNING,
(log_prefix + "xllm.log.WARNING.").c_str());
google::SetLogDestination(google::ERROR,
(log_prefix + "xllm.log.ERROR.").c_str());
google::SetStderrLogging(google::FATAL);
g_glog_inited.store(true, std::memory_order_release);
}
pthread_mutex_unlock(&g_log_init_mutex);
}
void shutdown_log() {
if (!g_glog_inited.load(std::memory_order_acquire)) {
return;
}
pthread_mutex_lock(&g_log_init_mutex);
if (g_glog_inited.load(std::memory_order_relaxed)) {
google::ShutdownGoogleLogging();
g_glog_inited.store(false, std::memory_order_release);
}
pthread_mutex_unlock(&g_log_init_mutex);
}
void set_init_options(BackendType backend_type,
const XLLM_InitOptions* init_options,
XLLM_InitOptions* xllm_init_options) {
if (init_options == nullptr) {
if (backend_type == BackendType::LLM) {
memcpy(xllm_init_options,
&XLLM_INIT_LLM_OPTIONS_DEFAULT,
sizeof(XLLM_InitOptions));
} else if (backend_type == BackendType::REC) {
memcpy(xllm_init_options,
&XLLM_INIT_REC_OPTIONS_DEFAULT,
sizeof(XLLM_InitOptions));
}
} else {
memcpy(xllm_init_options, init_options, sizeof(XLLM_InitOptions));
}
return;
}
void transfer_request_params(InferenceType inference_type,
const XLLM_RequestParams* request_params,
xllm::RequestParams* xllm_request_params) {
XLLM_RequestParams final_request_params;
if (nullptr == request_params) {
if (inference_type == InferenceType::LLM_COMPLETIONS ||
inference_type == InferenceType::LLM_CHAT_COMPLETIONS) {
memcpy(&final_request_params,
&XLLM_LLM_REQUEST_PARAMS_DEFAULT,
sizeof(XLLM_RequestParams));
} else if (inference_type == InferenceType::REC_COMPLETIONS ||
inference_type == InferenceType::REC_CHAT_COMPLETIONS) {
memcpy(&final_request_params,
&XLLM_REC_REQUEST_PARAMS_DEFAULT,
sizeof(XLLM_RequestParams));
}
} else {
memcpy(&final_request_params, request_params, sizeof(XLLM_RequestParams));
}
xllm_request_params->echo = final_request_params.echo;
xllm_request_params->offline = final_request_params.offline;
xllm_request_params->logprobs = final_request_params.logprobs;
xllm_request_params->ignore_eos = final_request_params.ignore_eos;
xllm_request_params->best_of = final_request_params.best_of;
xllm_request_params->top_k = final_request_params.top_k;
xllm_request_params->top_p = final_request_params.top_p;
xllm_request_params->n = final_request_params.n;
xllm_request_params->max_tokens = final_request_params.max_tokens;
xllm_request_params->frequency_penalty =
final_request_params.frequency_penalty;
xllm_request_params->presence_penalty = final_request_params.presence_penalty;
xllm_request_params->repetition_penalty =
final_request_params.repetition_penalty;
xllm_request_params->beam_width = final_request_params.beam_width;
xllm_request_params->num_return_sequences =
final_request_params.num_return_sequences;
xllm_request_params->top_logprobs = final_request_params.top_logprobs;
xllm_request_params->temperature = final_request_params.temperature;
xllm_request_params->request_id = final_request_params.request_id;
xllm_request_params->ttlt_slo_ms = final_request_params.ttlt_slo_ms;
xllm_request_params->ttft_slo_ms = final_request_params.ttft_slo_ms;
xllm_request_params->tpot_slo_ms = final_request_params.tpot_slo_ms;
return;
}
XLLM_Response* build_error_response(const std::string& request_id,
XLLM_StatusCode status_code,
const std::string& error_info) {
XLLM_Response* response = new XLLM_Response();
CHECK(nullptr != response);
response->status_code = status_code;
strncpy(
response->error_info, error_info.c_str(), XLLM_ERROR_INFO_MAX_LEN - 1);
response->error_info[XLLM_ERROR_INFO_MAX_LEN - 1] = '\0';
XLLM_SET_META_STRING_FIELD(response->id, request_id);
LOG(ERROR) << "Request [" << request_id << "] error: " << error_info
<< " (code: " << static_cast<int>(response->status_code) << ")";
return response;
}
XLLM_Response* build_success_response(const InferenceType& inference_type,
const RequestOutput& output,
RecPipelineType rec_pipeline_type,
const std::string& request_id,
int64_t created_time,
const std::string& model) {
XLLM_Response* response = new XLLM_Response();
CHECK(nullptr != response);
response->status_code = XLLM_StatusCode::kSuccess;
response->created = created_time;
XLLM_SET_META_STRING_FIELD(response->id, request_id);
XLLM_SET_META_STRING_FIELD(response->model, model);
if (inference_type == InferenceType::LLM_COMPLETIONS ||
inference_type == InferenceType::REC_COMPLETIONS) {
snprintf(response->object, sizeof(response->object), "text_completion");
} else if (inference_type == InferenceType::LLM_CHAT_COMPLETIONS ||
inference_type == InferenceType::REC_CHAT_COMPLETIONS) {
snprintf(response->object, sizeof(response->object), "chat.completion");
}
response->choices.entries_size = output.outputs.size();
response->choices.entries = new XLLM_Choice[response->choices.entries_size]();
CHECK(nullptr != response->choices.entries);
const bool is_rec_inference =
inference_type == InferenceType::REC_COMPLETIONS ||
inference_type == InferenceType::REC_CHAT_COMPLETIONS;
const bool is_onerec_pipeline =
is_rec_inference && is_onerec_pipeline_type(rec_pipeline_type);
if (is_onerec_pipeline) {
response->rec_outputs.entries_size = output.outputs.size();
response->rec_outputs.entries =
new XLLM_RecOutput[response->rec_outputs.entries_size]();
CHECK(nullptr != response->rec_outputs.entries);
}
int32_t total_item_count = 0;
const int32_t total_threshold = FLAGS_total_conversion_threshold;
for (int i = 0; i < output.outputs.size(); i++) {
const auto& seq_output = output.outputs[i];
XLLM_Choice& choice = response->choices.entries[i];
choice.index = seq_output.index;
XLLM_RecOutput* rec_output = nullptr;
if (response->rec_outputs.entries != nullptr) {
rec_output = &response->rec_outputs.entries[i];
rec_output->index = seq_output.index;
}
if (inference_type == InferenceType::LLM_COMPLETIONS ||
inference_type == InferenceType::REC_COMPLETIONS) {
size_t text_len = seq_output.text.length();
choice.text = new char[text_len + 1];
CHECK(nullptr != choice.text);
strncpy(choice.text, seq_output.text.c_str(), text_len + 1);
choice.text[text_len] = '\0';
} else if (inference_type == InferenceType::LLM_CHAT_COMPLETIONS ||
inference_type == InferenceType::REC_CHAT_COMPLETIONS) {
choice.message = new XLLM_ChatMessage();
CHECK(nullptr != choice.message);
snprintf(choice.message->role, sizeof(choice.message->role), "assistant");
size_t text_len = seq_output.text.length();
choice.message->content = new char[text_len + 1];
CHECK(nullptr != choice.message->content);
strncpy(choice.message->content, seq_output.text.c_str(), text_len + 1);
choice.message->content[text_len] = '\0';
}
if (seq_output.finish_reason.has_value()) {
XLLM_SET_META_STRING_FIELD(choice.finish_reason,
seq_output.finish_reason.value());
}
if (seq_output.token_ids.size() > 0) {
choice.token_size = seq_output.token_ids.size();
choice.token_ids = new int32_t[choice.token_size];
CHECK(nullptr != choice.token_ids);
for (int j = 0; j < choice.token_size; j++) {
choice.token_ids[j] = seq_output.token_ids[j];
}
}
if (seq_output.logprobs.has_value()) {
choice.logprobs.entries_size = seq_output.logprobs.value().size();
choice.logprobs.entries =
new XLLM_LogProb[choice.logprobs.entries_size]();
CHECK(nullptr != choice.logprobs.entries);
for (int j = 0; j < seq_output.logprobs.value().size(); j++) {
const auto& logprob = seq_output.logprobs.value()[j];
XLLM_LogProb& xllm_logprob = choice.logprobs.entries[j];
xllm_logprob.token_id = logprob.token_id;
xllm_logprob.logprob = logprob.logprob;
}
}
if (is_onerec_pipeline && FLAGS_enable_convert_tokens_to_item &&
rec_output != nullptr) {
size_t copied_item_count = 0;
if (!seq_output.item_ids_list.empty()) {
copied_item_count =
std::min(seq_output.item_ids_list.size(),
static_cast<size_t>(
std::max(total_threshold - total_item_count, 0)));
if (copied_item_count > 0) {
rec_output->item_ids_size = copied_item_count;
rec_output->item_ids = new int64_t[copied_item_count];
CHECK(nullptr != rec_output->item_ids);
for (size_t j = 0; j < copied_item_count; ++j) {
rec_output->item_ids[j] = seq_output.item_ids_list[j];
}
total_item_count += static_cast<int32_t>(copied_item_count);
}
} else if (seq_output.item_ids.has_value() &&
total_item_count < total_threshold) {
rec_output->item_ids_size = 1;
rec_output->item_ids = new int64_t[1];
CHECK(nullptr != rec_output->item_ids);
rec_output->item_ids[0] = seq_output.item_ids.value();
++total_item_count;
}
}
if (is_onerec_pipeline && FLAGS_enable_output_sku_logprobs &&
!seq_output.token_ids_logprobs.empty() && rec_output != nullptr) {
rec_output->rec_token_logprobs_size =
seq_output.token_ids_logprobs.size();
rec_output->rec_token_logprobs =
new float[rec_output->rec_token_logprobs_size];
CHECK(nullptr != rec_output->rec_token_logprobs);
for (size_t j = 0; j < rec_output->rec_token_logprobs_size; ++j) {
if (seq_output.token_ids_logprobs[j].has_value()) {
rec_output->rec_token_logprobs[j] =
seq_output.token_ids_logprobs[j].value();
} else {
rec_output->rec_token_logprobs[j] = 0.0f;
}
}
}
}
if (output.usage.has_value()) {
const auto& usage = output.usage.value();
response->usage.prompt_tokens = usage.num_prompt_tokens;
response->usage.completion_tokens = usage.num_generated_tokens;
response->usage.total_tokens = usage.num_total_tokens;
}
return response;
}
template <typename HandlerType, typename InputType>
XLLM_Response* handle_inference_request(
HandlerType* handler,
InferenceType inference_type,
const std::string& model_id,
const InputType& input,
void* extra,
uint32_t timeout_ms,
const XLLM_RequestParams* request_params) {
CHECK(nullptr != handler);
std::string request_id;
if (nullptr != request_params && strlen(request_params->request_id) > 0) {
request_id = request_params->request_id;
} else {
request_id = generate_request_id();
}
if (!handler->initialized) {
return build_error_response(
request_id, XLLM_StatusCode::kNotInitialized, "LLM is not initialized");
}
if (std::find(handler->model_ids.begin(),
handler->model_ids.end(),
model_id) == handler->model_ids.end()) {
return build_error_response(request_id,
XLLM_StatusCode::kModelNotFound,
"Specified model ID not loaded: " + model_id);
}
xllm::RequestParams xllm_request_params;
transfer_request_params(inference_type, request_params, &xllm_request_params);
xllm_request_params.request_id = request_id;
RecPipelineType rec_pipeline_type = RecPipelineType::kLlmRecDefault;
if constexpr (std::is_same_v<HandlerType, XLLM_REC_Handler>) {
rec_pipeline_type = handler->pipeline_type;
if (FLAGS_enable_output_sku_logprobs &&
is_onerec_pipeline_type(rec_pipeline_type)) {
xllm_request_params.logprobs = true;
}
}
const int64_t created_time = absl::ToUnixSeconds(absl::Now());
try {
auto promise_ptr = std::make_shared<folly::Promise<XLLM_Response*>>();
auto future = promise_ptr->getSemiFuture();
auto on_request_complete = [model_id,
request_id,
created_time,
inference_type,
rec_pipeline_type,
weak_promise = std::weak_ptr(promise_ptr)](
const RequestOutput& req_output) -> bool {
if (auto locked_promise = weak_promise.lock()) {
try {
if (req_output.status.has_value()) {
if (req_output.status.value().ok()) {
locked_promise->setValue(build_success_response(inference_type,
req_output,
rec_pipeline_type,
request_id,
created_time,
model_id));
} else {
locked_promise->setValue(build_error_response(
request_id,
XLLM_StatusCode::kInternalError,
"RequestOutput status is not ok, message: " +
req_output.status.value().message()));
}
} else {
locked_promise->setValue(
build_error_response(request_id,
XLLM_StatusCode::kInternalError,
"RequestOutput status has no value"));
}
return true;
} catch (const std::exception& e) {
LOG(ERROR) << "Build response failed: " << e.what();
locked_promise->setValue(build_error_response(
request_id,
XLLM_StatusCode::kInternalError,
"Build response failed: " + std::string(e.what())));
}
}
return false;
};
if constexpr (std::is_same_v<HandlerType, XLLM_LLM_Handler>) {
handler->master->handle_request(input,
std::nullopt,
xllm_request_params,
std::nullopt,
on_request_complete);
} else if constexpr (std::is_same_v<HandlerType, XLLM_REC_Handler>) {
if constexpr (std::is_same_v<InputType, std::vector<int>>) {
if (nullptr != extra) {
xllm::MMData* mm_data =
dynamic_cast<xllm::MMData*>(static_cast<xllm::MMData*>(extra));
CHECK(nullptr != mm_data);
std::optional<xllm::MMData> opt_mm_data = std::move(*mm_data);
handler->master->handle_request(
input, opt_mm_data, xllm_request_params, on_request_complete);
} else {
handler->master->handle_request("",
input,
std::nullopt,
xllm_request_params,
on_request_complete);
}
} else {
handler->master->handle_request(input,
std::nullopt,
std::nullopt,
xllm_request_params,
on_request_complete);
}
} else {
CHECK(false);
}
return std::move(future)
.via(handler->executor.get())
.within(std::chrono::milliseconds(timeout_ms))
.thenTry([request_id](
folly::Try<XLLM_Response*>&& result) -> XLLM_Response* {
if (result.hasValue()) return std::move(result).value();
std::string error_msg;
XLLM_StatusCode code = XLLM_StatusCode::kInternalError;
try {
result.throwUnlessValue();
} catch (const folly::FutureTimeout& e) {
error_msg = "Request timed out: " + std::string(e.what());
code = XLLM_StatusCode::kTimeout;
} catch (const std::exception& e) {
error_msg = "Inference failed: " + std::string(e.what());
} catch (...) {
error_msg = "Inference failed with unknown exception";
}
return build_error_response(request_id, code, error_msg);
})
.get();
} catch (...) {
return build_error_response(request_id,
XLLM_StatusCode::kInternalError,
"Critical error in inference pipeline");
}
}
void xllm_free_response(XLLM_Response* resp) {
if (nullptr == resp) {
return;
}
if (nullptr != resp->choices.entries) {
for (int i = 0; i < resp->choices.entries_size; ++i) {
XLLM_Choice& choice = resp->choices.entries[i];
if (nullptr != choice.text) {
delete[] choice.text;
choice.text = nullptr;
}
if (nullptr != choice.message) {
if (nullptr != choice.message->content) {
delete[] choice.message->content;
choice.message->content = nullptr;
}
delete choice.message;
choice.message = nullptr;
}
if (nullptr != choice.token_ids) {
delete[] choice.token_ids;
choice.token_ids = nullptr;
choice.token_size = 0;
}
if (nullptr != choice.logprobs.entries) {
delete[] choice.logprobs.entries;
choice.logprobs.entries = nullptr;
}
choice.logprobs.entries_size = 0;
}
delete[] resp->choices.entries;
resp->choices.entries = nullptr;
}
resp->choices.entries_size = 0;
if (nullptr != resp->rec_outputs.entries) {
for (size_t i = 0; i < resp->rec_outputs.entries_size; ++i) {
XLLM_RecOutput& rec_output = resp->rec_outputs.entries[i];
if (nullptr != rec_output.item_ids) {
delete[] rec_output.item_ids;
rec_output.item_ids = nullptr;
rec_output.item_ids_size = 0;
}
if (nullptr != rec_output.rec_token_logprobs) {
delete[] rec_output.rec_token_logprobs;
rec_output.rec_token_logprobs = nullptr;
rec_output.rec_token_logprobs_size = 0;
}
}
delete[] resp->rec_outputs.entries;
resp->rec_outputs.entries = nullptr;
}
resp->rec_outputs.entries_size = 0;
delete resp;
return;
}
torch::ScalarType xllm_dtype_to_torch_scalar_type(XLLM_DataType dtype) {
switch (dtype) {
case XLLM_DTYPE_UNDEFINED:
throw std::runtime_error(
"XLLM_DTYPE_UNDEFINED is not a valid dtype for tensor conversion");
case XLLM_DTYPE_FLOAT16:
return torch::kFloat16;
case XLLM_DTYPE_FLOAT32:
return torch::kFloat32;
case XLLM_DTYPE_FLOAT64:
return torch::kFloat64;
case XLLM_DTYPE_BFLOAT16:
return torch::kBFloat16;
case XLLM_DTYPE_INT8:
return torch::kInt8;
case XLLM_DTYPE_INT16:
return torch::kInt16;
case XLLM_DTYPE_INT32:
return torch::kInt32;
case XLLM_DTYPE_INT64:
return torch::kInt64;
case XLLM_DTYPE_BOOL:
return torch::kBool;
case XLLM_DTYPE_STRING:
throw std::runtime_error(
"String dtype is not supported for torch::Tensor");
default:
throw std::runtime_error("Unsupported XLLM_DataType: " +
std::to_string(dtype));
}
}
torch::Tensor convert_xllm_tensor_to_torch(const XLLM_Tensor& xllm_tensor) {
if (xllm_tensor.data == nullptr) {
throw std::runtime_error("XLLM_Tensor data pointer is null");
}
torch::ScalarType scalar_type =
xllm_dtype_to_torch_scalar_type(xllm_tensor.dtype);
std::vector<int64_t> shape;
for (int i = 0; i < xllm_tensor.dims.rank; ++i) {
int dim = xllm_tensor.dims.dim[i];
if (dim > 0) {
shape.push_back(dim);
}
}
if (shape.empty()) {
throw std::runtime_error("XLLM_Tensor all dimensions are invalid value");
}
torch::Tensor tensor =
torch::from_blob(const_cast<void*>(xllm_tensor.data), shape, scalar_type)
.clone();
return tensor;
}
xllm::MMDataItem convert_xllm_mm_item_to_internal(
const XLLM_MM_Item& xllm_item) {
uint32_t xllm_type_val = static_cast<uint32_t>(xllm_item.type);
xllm::MMType::Value internal_val = xllm::MMType::NONE;
switch (xllm_type_val) {
case XLLM_MM_TYPE_EMBEDDING:
internal_val = xllm::MMType::EMBEDDING;
break;
case XLLM_MM_TYPE_IMAGE:
internal_val = xllm::MMType::IMAGE;
break;
case XLLM_MM_TYPE_VIDEO:
internal_val = xllm::MMType::VIDEO;
break;
case XLLM_MM_TYPE_AUDIO:
internal_val = xllm::MMType::AUDIO;
break;
case XLLM_MM_TYPE_NONE:
internal_val = xllm::MMType::NONE;
break;
default:
throw std::runtime_error(std::string("Unsupported XLLM_MM_Type: ") +
std::to_string(xllm_type_val));
}
xllm::MMType item_type(internal_val);
xllm::MMDataItem internal_item(item_type);
xllm::MMItemState& state = internal_item.mutable_state();
xllm::MMItemState::TokenPos& token_pos = state.mutable_token_pos();
token_pos.offset = xllm_item.state.token_pos.offset;
token_pos.length = xllm_item.state.token_pos.length;
if (xllm_item.data.is_single_tensor) {
torch::Tensor tensor =
convert_xllm_tensor_to_torch(xllm_item.data.data.tensor);
internal_item.add("tensor", tensor);
} else {
std::vector<torch::Tensor> tensor_list;
const XLLM_Tensors& xllm_tensors = xllm_item.data.data.tensors;
for (size_t i = 0; i < xllm_tensors.entries_size; ++i) {
tensor_list.push_back(
convert_xllm_tensor_to_torch(xllm_tensors.entries[i]));
}
internal_item.add("tensor_list", tensor_list);
}
return internal_item;
}
bool convert_xllm_mm_data_to_internal(const XLLM_MM_Data* mm_data,
xllm::MMData& internal_mm_data) {
if (mm_data == nullptr || mm_data->type_mask == XLLM_MM_TYPE_NONE) {
return false;
}
xllm::MMType::Value internal_val =
static_cast<xllm::MMType::Value>(mm_data->type_mask);
xllm::MMType mm_type(internal_val);
if (mm_data->is_dict) {
const XLLM_MM_Dict& xllm_dict = mm_data->data.dict;
xllm::MMDict internal_dict;
for (size_t i = 0; i < xllm_dict.entries_size; ++i) {
const XLLM_MM_DictEntry& xllm_entry = xllm_dict.entries[i];
xllm::MMKey key(xllm_entry.key);
const XLLM_MM_Value& xllm_value = xllm_entry.value;
if (xllm_value.is_single_tensor) {
torch::Tensor tensor =
convert_xllm_tensor_to_torch(xllm_value.data.tensor);
internal_dict.insert({key, tensor});
} else {
std::vector<torch::Tensor> tensor_list;
const XLLM_Tensors& xllm_tensors = xllm_value.data.tensors;
for (size_t j = 0; j < xllm_tensors.entries_size; ++j) {
tensor_list.push_back(
convert_xllm_tensor_to_torch(xllm_tensors.entries[j]));
}
internal_dict.insert({key, tensor_list});
}
}
internal_mm_data.set<xllm::MMDict>(mm_type, internal_dict);
} else {
const XLLM_MM_Items& xllm_items = mm_data->data.items;
xllm::MMItemVec internal_item_vec;
for (size_t i = 0; i < xllm_items.entries_size; ++i) {
const XLLM_MM_Item& xllm_item = xllm_items.entries[i];
xllm::MMDataItem internal_item =
convert_xllm_mm_item_to_internal(xllm_item);
internal_item_vec.push_back(std::move(internal_item));
}
internal_mm_data.set<xllm::MMItemVec>(mm_type, internal_item_vec);
}
return true;
}
// 1. LLM Handler + const char* (text completions)
template XLLM_Response* handle_inference_request<XLLM_LLM_Handler, const char*>(
XLLM_LLM_Handler* handler,
InferenceType inference_type,
const std::string& model_id,
const char* const& input,
void* extra,
uint32_t timeout_ms,
const XLLM_RequestParams* request_params);
// 2. LLM Handler + std::vector<xllm::Message> (chat completions)
template XLLM_Response*
handle_inference_request<XLLM_LLM_Handler, std::vector<xllm::Message>>(
XLLM_LLM_Handler* handler,
InferenceType inference_type,
const std::string& model_id,
const std::vector<xllm::Message>& input,
void* extra,
uint32_t timeout_ms,
const XLLM_RequestParams* request_params);
// 3. REC Handler + const char* (REC completions)
template XLLM_Response* handle_inference_request<XLLM_REC_Handler, const char*>(
XLLM_REC_Handler* handler,
InferenceType inference_type,
const std::string& model_id,
const char* const& input,
void* extra,
uint32_t timeout_ms,
const XLLM_RequestParams* request_params);
// 4. REC Handler + std::vector<xllm::Message> (REC chat completions)
template XLLM_Response*
handle_inference_request<XLLM_REC_Handler, std::vector<xllm::Message>>(
XLLM_REC_Handler* handler,
InferenceType inference_type,
const std::string& model_id,
const std::vector<xllm::Message>& input,
void* extra,
uint32_t timeout_ms,
const XLLM_RequestParams* request_params);
// 5. REC Handler + std::vector<int> (chat completions)
template XLLM_Response*
handle_inference_request<XLLM_REC_Handler, std::vector<int>>(
XLLM_REC_Handler* handler,
InferenceType inference_type,
const std::string& model_id,
const std::vector<int>& input,
void* extra,
uint32_t timeout_ms,
const XLLM_RequestParams* request_params);
} // namespace helper
} // namespace xllm

172
upstream_ref/xllm/xllm/c_api/internal/helper.h Normal file
View File

@@ -0,0 +1,172 @@
/* Copyright 2025 The xLLM Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
https://github.com/jd-opensource/xllm/blob/main/LICENSE
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#pragma once
#include <absl/time/clock.h>
#include <absl/time/time.h>
#include <folly/executors/CPUThreadPoolExecutor.h>
#include <folly/futures/Future.h>
#include <folly/futures/Promise.h>
#include <glog/logging.h>
#include <algorithm>
#include <memory>
#include <string>
#include <vector>
#include "c_api/default.h"
#include "c_api/types.h"
#include "core/common/instance_name.h"
#include "core/distributed_runtime/llm_master.h"
#include "core/distributed_runtime/rec_master.h"
#include "core/framework/request/request_output.h"
#include "core/framework/request/request_params.h"
#include "core/util/rec_model_utils.h"
/**
* @brief Opaque handle for LLM inference instance
*/
struct XLLM_LLM_Handler {
/** Flag indicating if LLM instance is initialized and ready for inference */
bool initialized{false};
/** List of loaded model IDs (for model existence validation) */
std::vector<std::string> model_ids;
/** Core controller for LLM runtime management */
std::unique_ptr<xllm::LLMMaster> master;
/** Thread pool for asynchronous inference task scheduling */
std::unique_ptr<folly::CPUThreadPoolExecutor> executor;
};
/**
* @brief Opaque handle for REC (Recommendation) inference instance
*/
struct XLLM_REC_Handler {
/** Flag indicating if REC instance is initialized and ready for inference */
bool initialized{false};
/** Selected REC pipeline type for the loaded model */
xllm::RecPipelineType pipeline_type{xllm::RecPipelineType::kLlmRecDefault};
/** List of loaded recommendation model IDs */
std::vector<std::string> model_ids;
/** Core controller for REC runtime management */
std::unique_ptr<xllm::RecMaster> master;
/** Thread pool for asynchronous recommendation task scheduling */
std::unique_ptr<folly::CPUThreadPoolExecutor> executor;
};
namespace xllm {
namespace helper {
enum class BackendType { LLM = 0, VLM = 1, REC = 2 };
enum class InferenceType {
LLM_COMPLETIONS = 0,
LLM_CHAT_COMPLETIONS = 1,
REC_COMPLETIONS = 2,
REC_CHAT_COMPLETIONS = 3,
REC_TOKENID_COMPLETIONS = 4,
};
#define XLLM_SET_META_STRING_FIELD(DST, SRC_STR) \
do { \
static_assert(sizeof(DST) > 1, "Destination buffer is too small"); \
strncpy( \
(char*)(DST), (SRC_STR).c_str(), XLLM_META_STRING_FIELD_MAX_LEN - 1); \
(DST)[XLLM_META_STRING_FIELD_MAX_LEN - 1] = '\0'; \
} while (0)
/**
* @brief Thread-safe glog initialization for xLLM framework
* @note This API is idempotent (multiple calls have same effect as single call)
* @note Thread-safe: protected by pthread mutex to prevent race condition
* @param log_dir Directory to store log files (empty = current directory)
*/
void init_log(const std::string& log_dir);
/**
* @brief Safely shutdown glog and release resources
* @note Call this function before program exit (optional but recommended)
*/
void shutdown_log();
/**
* @brief Set init options, merge default options
*/
void set_init_options(BackendType backend_type,
const XLLM_InitOptions* init_options,
XLLM_InitOptions* xllm_init_options);
/**
* @brief Transfer C API request params to xLLM internal request params
*/
void transfer_request_params(InferenceType inference_type,
const XLLM_RequestParams* request_params,
xllm::RequestParams* xllm_request_params);
/**
* @brief Build error response for failed inference requests
*/
XLLM_Response* build_error_response(const std::string& request_id,
XLLM_StatusCode status_code,
const std::string& error_info);
/**
* @brief Build success response for completed inference requests
*/
XLLM_Response* build_success_response(const InferenceType& inference_type,
const xllm::RequestOutput& output,
xllm::RecPipelineType rec_pipeline_type,
const std::string& request_id,
int64_t created_time,
const std::string& model);
/**
* @brief Generic inference request handler (template function)
*/
template <typename HandlerType, typename InputType>
XLLM_Response* handle_inference_request(
HandlerType* handler,
InferenceType inference_type,
const std::string& model_id,
const InputType& input,
void* extra,
uint32_t timeout_ms,
const XLLM_RequestParams* request_params);
/**
* @brief Safely free all memory allocated in XLLM_Response
*/
void xllm_free_response(XLLM_Response* resp);
/**
* @brief Generate unique request ID for tracing
*/
std::string generate_request_id();
torch::ScalarType xllm_dtype_to_torch_scalar_type(XLLM_DataType dtype);
torch::Tensor convert_xllm_tensor_to_torch(const XLLM_Tensor& xllm_tensor);
xllm::MMDataItem convert_xllm_mm_item_to_internal(
const XLLM_MM_Item& xllm_item);
bool convert_xllm_mm_data_to_internal(const XLLM_MM_Data* mm_data,
xllm::MMData& internal_mm_data);
} // namespace helper
} // namespace xllm

221
upstream_ref/xllm/xllm/c_api/internal/llm.cpp Normal file
View File

@@ -0,0 +1,221 @@
/* Copyright 2025 The xLLM Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
https://github.com/jd-opensource/xllm/blob/main/LICENSE
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#include "c_api/llm.h"
#include <folly/Unit.h>
#include <folly/experimental/coro/Timeout.h>
#include <folly/futures/Future.h>
#include <glog/logging.h>
#include <pthread.h>
#include <atomic>
#include <cstring>
#include <exception>
#include <stdexcept>
#include "core/common/global_flags.h"
#include "helper.h"
XLLM_CAPI_EXPORT XLLM_LLM_Handler* xllm_llm_create(void) {
XLLM_LLM_Handler* handler = new XLLM_LLM_Handler();
CHECK(nullptr != handler);
handler->initialized = false;
return handler;
}
XLLM_CAPI_EXPORT void xllm_llm_destroy(XLLM_LLM_Handler* handler) {
if (!handler) return;
handler->master.reset();
handler->executor.reset();
handler->model_ids.clear();
handler->initialized = false;
delete handler;
}
XLLM_CAPI_EXPORT void xllm_llm_init_options_default(
XLLM_InitOptions* init_options) {
if (nullptr == init_options) return;
*init_options = XLLM_INIT_LLM_OPTIONS_DEFAULT;
}
XLLM_CAPI_EXPORT bool xllm_llm_initialize(
XLLM_LLM_Handler* handler,
const char* model_path,
const char* devices,
const XLLM_InitOptions* init_options) {
if (!handler || !model_path || !devices) return false;
try {
XLLM_InitOptions xllm_init_options;
xllm::helper::set_init_options(
xllm::helper::BackendType::LLM, init_options, &xllm_init_options);
std::string log_dir(xllm_init_options.log_dir);
if (!log_dir.empty()) {
xllm::helper::init_log(xllm_init_options.log_dir);
}
if (!std::filesystem::exists(model_path)) {
LOG(ERROR) << "model path[" << model_path << "] does not exist";
return false;
}
xllm::Options options;
options.model_path(model_path)
.task_type(xllm_init_options.task)
.devices(devices)
.draft_model_path(xllm_init_options.draft_model)
.draft_devices(xllm_init_options.draft_devices)
.backend("llm")
.block_size(xllm_init_options.block_size)
.max_cache_size(xllm_init_options.max_cache_size)
.max_memory_utilization(xllm_init_options.max_memory_utilization)
.enable_prefix_cache(xllm_init_options.enable_prefix_cache)
.max_tokens_per_batch(xllm_init_options.max_tokens_per_batch)
.max_seqs_per_batch(xllm_init_options.max_seqs_per_batch)
.max_tokens_per_chunk_for_prefill(
xllm_init_options.max_tokens_per_chunk_for_prefill)
.num_speculative_tokens(xllm_init_options.num_speculative_tokens)
.num_request_handling_threads(
xllm_init_options.num_request_handling_threads)
.communication_backend(xllm_init_options.communication_backend)
.expert_parallel_degree(xllm_init_options.expert_parallel_degree)
.enable_chunked_prefill(xllm_init_options.enable_chunked_prefill)
.enable_prefill_sp(xllm_init_options.enable_prefill_sp)
.master_node_addr(xllm_init_options.master_node_addr)
.device_ip(xllm_init_options.device_ip)
.transfer_listen_port(xllm_init_options.transfer_listen_port)
.nnodes(xllm_init_options.nnodes)
.node_rank(xllm_init_options.node_rank)
.dp_size(xllm_init_options.dp_size)
.ep_size(xllm_init_options.ep_size)
.instance_name(xllm_init_options.instance_name)
.enable_disagg_pd(xllm_init_options.enable_disagg_pd)
.enable_schedule_overlap(xllm_init_options.enable_schedule_overlap)
.enable_pd_ooc(xllm_init_options.enable_pd_ooc)
.kv_cache_transfer_mode(xllm_init_options.kv_cache_transfer_mode)
.enable_shm(xllm_init_options.enable_shm)
.is_local(true)
.server_idx(xllm_init_options.server_idx);
options.enable_graph(FLAGS_enable_graph);
#if !defined(USE_NPU) && !defined(USE_CUDA)
FLAGS_enable_block_copy_kernel = false;
#endif
handler->master = std::make_unique<xllm::LLMMaster>(options);
handler->master->run();
size_t cpu_cores = std::thread::hardware_concurrency();
size_t thread_num = std::clamp((cpu_cores == 0) ? 8 : cpu_cores / 2,
static_cast<size_t>(4),
static_cast<size_t>(16));
handler->executor =
std::make_unique<folly::CPUThreadPoolExecutor>(thread_num);
std::filesystem::path model_path_fs =
std::filesystem::path(model_path).lexically_normal();
std::string model_id;
if (model_path_fs.has_filename()) {
model_id = model_path_fs.filename().string();
} else if (!model_path_fs.empty()) {
model_id = model_path_fs.string();
} else {
model_id = "default";
}
handler->model_ids.clear();
handler->model_ids.emplace_back(model_id);
handler->initialized = true;
return true;
} catch (const std::exception& e) {
LOG(ERROR) << "LLM initialization failed: " << e.what();
}
handler->master.reset();
handler->executor.reset();
handler->model_ids.clear();
handler->initialized = false;
return false;
}
XLLM_CAPI_EXPORT void xllm_llm_request_params_default(
XLLM_RequestParams* request_params) {
if (nullptr == request_params) return;
*request_params = XLLM_LLM_REQUEST_PARAMS_DEFAULT;
}
XLLM_CAPI_EXPORT XLLM_Response* xllm_llm_completions(
XLLM_LLM_Handler* handler,
const char* model_id,
const char* prompt,
uint32_t timeout_ms,
const XLLM_RequestParams* request_params) {
if (!handler || !model_id || *model_id == '\0' || !prompt ||
*prompt == '\0') {
return xllm::helper::build_error_response(
"", XLLM_StatusCode::kInvalidRequest, "Invalid input parameters");
}
return xllm::helper::handle_inference_request(
handler,
xllm::helper::InferenceType::LLM_COMPLETIONS,
model_id,
prompt,
nullptr,
timeout_ms,
request_params);
}
XLLM_CAPI_EXPORT XLLM_Response* xllm_llm_chat_completions(
XLLM_LLM_Handler* handler,
const char* model_id,
const XLLM_ChatMessage* messages,
size_t messages_count,
uint32_t timeout_ms,
const XLLM_RequestParams* request_params) {
if (!handler || !model_id || *model_id == '\0' || !messages ||
messages_count == 0) {
return xllm::helper::build_error_response(
"", XLLM_StatusCode::kInvalidRequest, "Invalid input parameters");
}
std::vector<xllm::Message> xllm_messages;
xllm_messages.reserve(messages_count);
for (int i = 0; i < messages_count; i++) {
xllm_messages.emplace_back(messages[i].role, messages[i].content);
}
return xllm::helper::handle_inference_request(
handler,
xllm::helper::InferenceType::LLM_CHAT_COMPLETIONS,
model_id,
xllm_messages,
nullptr,
timeout_ms,
request_params);
}
XLLM_CAPI_EXPORT void xllm_llm_free_response(XLLM_Response* resp) {
return xllm::helper::xllm_free_response(resp);
}

432
upstream_ref/xllm/xllm/c_api/internal/rec.cpp Normal file
View File

@@ -0,0 +1,432 @@
/* Copyright 2025 The xLLM Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
https://github.com/jd-opensource/xllm/blob/main/LICENSE
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#include "c_api/rec.h"
#include <folly/Unit.h>
#include <folly/experimental/coro/Timeout.h>
#include <folly/futures/Future.h>
#include <glog/logging.h>
#include <pthread.h>
#include <atomic>
#include <cstring>
#include <exception>
#include <stdexcept>
#include "core/framework/model_loader.h"
#include "core/util/rec_model_utils.h"
#include "helper.h"
namespace {
const char* get_rec_pipeline_name(xllm::RecPipelineType pipeline_type) {
switch (pipeline_type) {
case xllm::RecPipelineType::kLlmRecDefault:
return "LlmRecEnginePipeline";
case xllm::RecPipelineType::kLlmRecWithMmData:
return "LlmRecWithMmData";
case xllm::RecPipelineType::kLlmRecMultiRoundPipeline:
return "RecMultiRoundEnginePipeline";
case xllm::RecPipelineType::kOneRecDefault:
return "OneRecPrefillOnlyEnginePipeline";
case xllm::RecPipelineType::kOneRecXAttentionPipeline:
return "OneRecXAttentionEnginePipeline";
default:
return "UnknownRecPipeline";
}
}
void reset_pipeline_runtime_toggles() {
FLAGS_enable_rec_fast_sampler = false;
FLAGS_enable_prefill_piecewise_graph = false;
FLAGS_enable_xattention_one_stage = false;
FLAGS_enable_graph_mode_decode_no_padding = false;
FLAGS_enable_rec_prefill_only = false;
FLAGS_enable_constrained_decoding = false;
FLAGS_enable_topk_sorted = false;
}
void apply_multi_round_pipeline_toggles() {
FLAGS_enable_rec_fast_sampler = true;
FLAGS_enable_prefill_piecewise_graph = true;
FLAGS_enable_xattention_one_stage = false;
FLAGS_enable_graph_mode_decode_no_padding = true;
FLAGS_enable_topk_sorted = false;
}
void apply_onerec_pipeline_toggles(xllm::Options* options) {
const bool enable_onerec_xattention = FLAGS_max_decode_rounds > 0;
FLAGS_enable_rec_prefill_only = !enable_onerec_xattention;
FLAGS_enable_constrained_decoding = true;
FLAGS_enable_prefix_cache = false;
FLAGS_enable_schedule_overlap = false;
FLAGS_enable_chunked_prefill = false;
options->enable_prefix_cache(false)
.enable_schedule_overlap(false)
.enable_chunked_prefill(false);
if (!enable_onerec_xattention) {
// Legacy OneRec keeps the historical fixed decode-step behavior.
FLAGS_max_decode_rounds = 0;
}
}
} // namespace
XLLM_CAPI_EXPORT XLLM_REC_Handler* xllm_rec_create(void) {
XLLM_REC_Handler* handler = new XLLM_REC_Handler();
CHECK(nullptr != handler);
handler->initialized = false;
handler->pipeline_type = xllm::RecPipelineType::kLlmRecDefault;
return handler;
}
XLLM_CAPI_EXPORT void xllm_rec_destroy(XLLM_REC_Handler* handler) {
if (!handler) return;
handler->master.reset();
handler->executor.reset();
handler->model_ids.clear();
handler->pipeline_type = xllm::RecPipelineType::kLlmRecDefault;
handler->initialized = false;
delete handler;
}
XLLM_CAPI_EXPORT void xllm_rec_init_options_default(
XLLM_InitOptions* init_options) {
if (nullptr == init_options) return;
*init_options = XLLM_INIT_REC_OPTIONS_DEFAULT;
}
XLLM_CAPI_EXPORT bool xllm_rec_initialize(
XLLM_REC_Handler* handler,
const char* model_path,
const char* devices,
const XLLM_InitOptions* init_options) {
if (!handler || !model_path || !devices) return false;
try {
XLLM_InitOptions xllm_init_options;
xllm::helper::set_init_options(
xllm::helper::BackendType::REC, init_options, &xllm_init_options);
std::string log_dir(xllm_init_options.log_dir);
if (!log_dir.empty()) {
xllm::helper::init_log(xllm_init_options.log_dir);
}
if (!std::filesystem::exists(model_path)) {
LOG(ERROR) << "model path[" << model_path << "] does not exist";
return false;
}
xllm::Options options;
options.model_path(model_path)
.task_type(xllm_init_options.task)
.devices(devices)
.draft_model_path(xllm_init_options.draft_model)
.draft_devices(xllm_init_options.draft_devices)
.backend("rec")
.block_size(xllm_init_options.block_size)
.max_cache_size(xllm_init_options.max_cache_size)
.max_memory_utilization(xllm_init_options.max_memory_utilization)
.enable_prefix_cache(xllm_init_options.enable_prefix_cache)
.max_tokens_per_batch(xllm_init_options.max_tokens_per_batch)
.max_seqs_per_batch(xllm_init_options.max_seqs_per_batch)
.max_tokens_per_chunk_for_prefill(
xllm_init_options.max_tokens_per_chunk_for_prefill)
.num_speculative_tokens(xllm_init_options.num_speculative_tokens)
.num_request_handling_threads(
xllm_init_options.num_request_handling_threads)
.communication_backend(xllm_init_options.communication_backend)
.expert_parallel_degree(xllm_init_options.expert_parallel_degree)
.enable_chunked_prefill(xllm_init_options.enable_chunked_prefill)
.master_node_addr(xllm_init_options.master_node_addr)
.device_ip(xllm_init_options.device_ip)
.transfer_listen_port(xllm_init_options.transfer_listen_port)
.nnodes(xllm_init_options.nnodes)
.node_rank(xllm_init_options.node_rank)
.dp_size(xllm_init_options.dp_size)
.ep_size(xllm_init_options.ep_size)
.instance_name(xllm_init_options.instance_name)
.enable_disagg_pd(xllm_init_options.enable_disagg_pd)
.enable_schedule_overlap(xllm_init_options.enable_schedule_overlap)
.enable_pd_ooc(xllm_init_options.enable_pd_ooc)
.kv_cache_transfer_mode(xllm_init_options.kv_cache_transfer_mode)
.enable_shm(xllm_init_options.enable_shm)
.is_local(true)
.server_idx(xllm_init_options.server_idx);
// @TODO: Currently, gflags are configured through hard coding, which needs
// to be improved in the future. For example, a separate gflags
// configuration file can be provided to the so for setting gflags.
//
// REC so still has two configuration paths:
// - some request/runtime code reads FLAGS_* directly
// - master/worker construction reads xllm::Options
//
// The fields copied from init options below are read from FLAGS_* today.
// beam_width/block_size/max_tokens/max_seqs are also represented in
// Options, so duplicated values must stay aligned.
FLAGS_beam_width = xllm_init_options.beam_width;
FLAGS_max_decode_rounds = xllm_init_options.max_decode_rounds;
FLAGS_max_seqs_per_batch = xllm_init_options.max_seqs_per_batch;
FLAGS_max_tokens_per_batch = xllm_init_options.max_tokens_per_batch;
FLAGS_block_size = xllm_init_options.block_size;
FLAGS_enable_rec_prefill_only = xllm_init_options.enable_rec_prefill_only;
FLAGS_enable_prefix_cache = xllm_init_options.enable_prefix_cache;
FLAGS_enable_schedule_overlap = xllm_init_options.enable_schedule_overlap;
FLAGS_enable_chunked_prefill = xllm_init_options.enable_chunked_prefill;
FLAGS_enable_graph = xllm_init_options.enable_graph;
FLAGS_rec_worker_max_concurrency =
xllm_init_options.rec_worker_max_concurrency;
FLAGS_enable_block_copy_kernel = xllm_init_options.enable_block_copy_kernel;
auto model_loader = xllm::ModelLoader::create(model_path);
if (model_loader == nullptr) {
LOG(ERROR) << "Failed to create model loader for path: " << model_path;
return false;
}
const auto& model_args = model_loader->model_args();
const xllm::RecModelKind rec_model_kind =
xllm::get_rec_model_kind(model_args.model_type());
if (rec_model_kind == xllm::RecModelKind::kNone) {
LOG(ERROR) << "Unsupported rec model_type: " << model_args.model_type();
return false;
}
const xllm::RecPipelineType pipeline_type =
xllm::get_rec_pipeline_type(rec_model_kind);
// Pipeline-specific runtime toggles in the REC so path.
reset_pipeline_runtime_toggles();
switch (pipeline_type) {
case xllm::RecPipelineType::kLlmRecMultiRoundPipeline:
apply_multi_round_pipeline_toggles();
break;
case xllm::RecPipelineType::kOneRecDefault:
case xllm::RecPipelineType::kOneRecXAttentionPipeline:
apply_onerec_pipeline_toggles(&options);
break;
case xllm::RecPipelineType::kLlmRecDefault:
case xllm::RecPipelineType::kLlmRecWithMmData:
break;
default:
LOG(ERROR) << "Unsupported rec pipeline type: "
<< static_cast<int32_t>(pipeline_type);
return false;
}
// Keep dual-source settings aligned with the FLAGS_* values above.
options.enable_graph(FLAGS_enable_graph)
.beam_width(FLAGS_beam_width)
.rec_worker_max_concurrency(FLAGS_rec_worker_max_concurrency);
LOG(INFO) << "REC C API selected pipeline="
<< get_rec_pipeline_name(pipeline_type)
<< ", model_type=" << model_args.model_type()
<< ", enable_rec_prefill_only=" << FLAGS_enable_rec_prefill_only
<< ", enable_constrained_decoding="
<< FLAGS_enable_constrained_decoding
<< ", enable_prefix_cache=" << FLAGS_enable_prefix_cache
<< ", enable_schedule_overlap=" << FLAGS_enable_schedule_overlap
<< ", enable_chunked_prefill=" << FLAGS_enable_chunked_prefill
<< ", enable_rec_fast_sampler=" << FLAGS_enable_rec_fast_sampler
<< ", max_decode_rounds=" << FLAGS_max_decode_rounds;
#if !defined(USE_NPU) && !defined(USE_CUDA)
FLAGS_enable_block_copy_kernel = false;
#endif
handler->master = std::make_unique<xllm::RecMaster>(options);
handler->master->run();
size_t cpu_cores = std::thread::hardware_concurrency();
size_t thread_num = std::clamp((cpu_cores == 0) ? 8 : cpu_cores / 2,
static_cast<size_t>(4),
static_cast<size_t>(16));
handler->executor =
std::make_unique<folly::CPUThreadPoolExecutor>(thread_num);
std::filesystem::path model_path_fs =
std::filesystem::path(model_path).lexically_normal();
std::string model_id;
if (model_path_fs.has_filename()) {
model_id = model_path_fs.filename().string();
} else if (!model_path_fs.empty()) {
model_id = model_path_fs.string();
} else {
model_id = "default";
}
handler->model_ids.clear();
handler->model_ids.emplace_back(model_id);
handler->pipeline_type = pipeline_type;
handler->initialized = true;
return true;
} catch (const std::exception& e) {
LOG(ERROR) << "LLM initialization failed: " << e.what();
}
handler->master.reset();
handler->executor.reset();
handler->model_ids.clear();
handler->pipeline_type = xllm::RecPipelineType::kLlmRecDefault;
handler->initialized = false;
return false;
}
XLLM_CAPI_EXPORT void xllm_rec_request_params_default(
XLLM_RequestParams* request_params) {
if (nullptr == request_params) return;
*request_params = XLLM_REC_REQUEST_PARAMS_DEFAULT;
}
XLLM_CAPI_EXPORT XLLM_Response* xllm_rec_text_completions(
XLLM_REC_Handler* handler,
const char* model_id,
const char* prompt,
uint32_t timeout_ms,
const XLLM_RequestParams* request_params) {
if (!handler || !model_id || *model_id == '\0' || !prompt ||
*prompt == '\0') {
return xllm::helper::build_error_response(
"", XLLM_StatusCode::kInvalidRequest, "Invalid input parameters");
}
return xllm::helper::handle_inference_request(
handler,
xllm::helper::InferenceType::REC_COMPLETIONS,
model_id,
prompt,
nullptr,
timeout_ms,
request_params);
}
XLLM_CAPI_EXPORT XLLM_Response* xllm_rec_token_completions(
XLLM_REC_Handler* handler,
const char* model_id,
const int32_t* token_ids,
size_t token_size,
uint32_t timeout_ms,
const XLLM_RequestParams* request_params) {
if (!handler || !model_id || *model_id == '\0' || !token_ids ||
token_size == 0) {
return xllm::helper::build_error_response(
"", XLLM_StatusCode::kInvalidRequest, "Invalid input parameters");
}
std::vector<int> token_ids_vec;
for (int i = 0; i < token_size; i++) {
token_ids_vec.push_back(token_ids[i]);
}
return xllm::helper::handle_inference_request(
handler,
xllm::helper::InferenceType::REC_COMPLETIONS,
model_id,
token_ids_vec,
nullptr,
timeout_ms,
request_params);
}
XLLM_CAPI_EXPORT XLLM_Response* xllm_rec_multimodal_completions(
XLLM_REC_Handler* handler,
const char* model_id,
const int32_t* token_ids,
size_t token_size,
const XLLM_MM_Data* mm_data,
uint32_t timeout_ms,
const XLLM_RequestParams* request_params) {
if (!handler || !model_id || *model_id == '\0' || !token_ids ||
token_size == 0) {
return xllm::helper::build_error_response(
"", XLLM_StatusCode::kInvalidRequest, "Invalid input parameters");
}
if (!mm_data) {
return xllm_rec_token_completions(
handler, model_id, token_ids, token_size, timeout_ms, request_params);
}
xllm::MMData internal_mm_data;
try {
bool ret = xllm::helper::convert_xllm_mm_data_to_internal(mm_data,
internal_mm_data);
if (!ret) {
return xllm::helper::build_error_response(
"", XLLM_StatusCode::kInternalError, "Fail in mm_data conversion");
}
} catch (const std::exception& e) {
return xllm::helper::build_error_response(
"",
XLLM_StatusCode::kInternalError,
"Critical error in mm_data conversion: " + std::string(e.what()));
}
std::vector<int> token_ids_vec;
for (int i = 0; i < token_size; i++) {
token_ids_vec.push_back(token_ids[i]);
}
return xllm::helper::handle_inference_request(
handler,
xllm::helper::InferenceType::REC_COMPLETIONS,
model_id,
token_ids_vec,
static_cast<void*>(&internal_mm_data),
timeout_ms,
request_params);
}
XLLM_CAPI_EXPORT XLLM_Response* xllm_rec_chat_completions(
XLLM_REC_Handler* handler,
const char* model_id,
const XLLM_ChatMessage* messages,
size_t messages_count,
uint32_t timeout_ms,
const XLLM_RequestParams* request_params) {
if (!handler || !model_id || *model_id == '\0' || !messages ||
messages_count == 0) {
return xllm::helper::build_error_response(
"", XLLM_StatusCode::kInvalidRequest, "Invalid input parameters");
}
std::vector<xllm::Message> xllm_messages;
xllm_messages.reserve(messages_count);
for (int i = 0; i < messages_count; i++) {
xllm_messages.emplace_back(messages[i].role, messages[i].content);
}
return xllm::helper::handle_inference_request(
handler,
xllm::helper::InferenceType::REC_CHAT_COMPLETIONS,
model_id,
xllm_messages,
nullptr,
timeout_ms,
request_params);
}
XLLM_CAPI_EXPORT void xllm_rec_free_response(XLLM_Response* resp) {
return xllm::helper::xllm_free_response(resp);
}

227
upstream_ref/xllm/xllm/c_api/llm.h Normal file
View File

@@ -0,0 +1,227 @@
/* Copyright 2025 The xLLM Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
https://github.com/jd-opensource/xllm/blob/main/LICENSE
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef XLLM_LLM_API_H
#define XLLM_LLM_API_H
#ifdef __cplusplus
extern "C" {
#endif
#include <stdbool.h>
#include <stddef.h>
#include <stdint.h>
#include "types.h"
/**
* @brief Opaque handle to an LLM inference instance
*
* This handle encapsulates all internal state of an LLM inference runtime,
* including model weights, device context, and generation cache.
* The handle MUST be created via xllm_llm_create() and destroyed via
* xllm_llm_destroy() to prevent memory/device resource leaks.
*/
typedef struct XLLM_LLM_Handler XLLM_LLM_Handler;
/**
* @brief Create a new LLM inference instance handle
*
* Allocates memory and initializes a new LLM handler with default internal
* state (empty model, uninitialized device context). This is the first function
* that must be called before using any other LLM APIs.
*
* @return Valid XLLM_LLM_Handler* on success; NULL if memory allocation fails
* @see xllm_llm_destroy
*/
XLLM_CAPI_EXPORT XLLM_LLM_Handler* xllm_llm_create(void);
/**
* @brief Destroy an LLM instance handle and release all associated resources
*
* Frees all memory allocated for the LLM instance, including:
* - Model weights (host/device memory)
* - Runtime context (CUDA/NPU streams, compute graphs)
* - Generation cache and temporary buffers
* - Device resources (contexts, queues)
*
* This function is idempotent—calling with NULL has no effect.
*
* @param handler LLM instance handle (NULL = no operation)
* @note Mandatory: Must be called to avoid memory/device resource leaks
* @see xllm_llm_create
*/
XLLM_CAPI_EXPORT void xllm_llm_destroy(XLLM_LLM_Handler* handler);
/**
* @brief Initialize XLLM_InitOptions with canonical default values
*
* Populates the XLLM_InitOptions struct with industry-standard default values
*
* @param init_options Pointer to XLLM_InitOptions to initialize (NULL = no-op)
* @see xllm_llm_initialize, XLLM_INIT_LLM_OPTIONS_DEFAULT
*/
XLLM_CAPI_EXPORT void xllm_llm_init_options_default(
XLLM_InitOptions* init_options);
/**
* @brief Initialize the LLM model and runtime environment
*
* Loads model weights from the specified path, configures target devices,
* initializes compute contexts, and prepares the inference runtime.
* Must be called exactly once per handler before using completion/chat APIs.
*
* If init_options is NULL, this function automatically uses the default values
* from XLLM_INIT_LLM_OPTIONS_DEFAULT (via xllm_llm_init_options_default()).
*
* @param handler Valid LLM instance handle (must not be NULL)
* @param model_path Null-terminated string of the model directory/file path
* (supports .bin/.pth/.safetensors formats)
* @param devices Null-terminated string specifying target devices (format:
* "npu:0,1" (specific NPUs), "cuda:0" (single GPU), "auto"
* (automatic selection))
* @param init_options Advanced initialization options (NULL = use defaults)
*
* @return true if initialization succeeds; false on failure (see failure causes
* below)
*
* @failure_causes
* - Invalid handler (NULL or already destroyed)
* - Invalid model_path (non-existent, corrupted, or unsupported format)
* - Invalid devices string (malformed format or unavailable devices)
* - Model load error (mismatched model architecture or weight corruption)
* - Device initialization failure (out of memory, driver error)
*
* @see xllm_llm_init_options_default, XLLM_INIT_LLM_OPTIONS_DEFAULT,
* xllm_llm_create
*/
XLLM_CAPI_EXPORT bool xllm_llm_initialize(XLLM_LLM_Handler* handler,
const char* model_path,
const char* devices,
const XLLM_InitOptions* init_options);
/**
* @brief Initialize XLLM_RequestParams with canonical generation defaults
*
* Populates the XLLM_RequestParams struct with safe default generation values
*
* @param request_params Pointer to XLLM_RequestParams to initialize (NULL =
* no-op)
* @see xllm_llm_completions, xllm_llm_chat_completions,
* XLLM_LLM_REQUEST_PARAMS_DEFAULT
*/
XLLM_CAPI_EXPORT void xllm_llm_request_params_default(
XLLM_RequestParams* request_params);
/**
* @brief Generate text completions for a single prompt
*
* Generates continuation text for the input prompt using the initialized LLM
* model. Returns a dynamically allocated response struct that MUST be freed
* with xllm_llm_free_response() to avoid memory leaks.
*
* If request_params is NULL, this function automatically uses the default
* values from XLLM_LLM_REQUEST_PARAMS_DEFAULT (via
* xllm_llm_request_params_default()).
*
* @param handler Valid, initialized LLM instance handle (must not be NULL)
* @param model_id Null-terminated string of the loaded model ID (must match
* model_path)
* @param prompt Null-terminated string of input text to complete (non-empty)
* @param timeout_ms Timeout in milliseconds (0 = no timeout, wait indefinitely)
* @param request_params Generation parameters (NULL = use defaults)
*
* @return Pointer to XLLM_Response on success; NULL ONLY if memory allocation
* fails (response->status indicates the actual result status)
*
* @response_status_codes
* - kSuccess: Valid response generated (check response->choices for results)
* - kNotInitialized: Handler not initialized with xllm_llm_initialize()
* - kInvalidRequest: Invalid prompt (empty/NULL) or model_id (mismatch)
* - kTimeout: Generation exceeded timeout_ms (partial results may be available)
*
* @warning Mandatory: Call xllm_llm_free_response() to release response memory
* @see xllm_llm_request_params_default, XLLM_LLM_REQUEST_PARAMS_DEFAULT,
* xllm_llm_free_response
*/
XLLM_CAPI_EXPORT XLLM_Response* xllm_llm_completions(
XLLM_LLM_Handler* handler,
const char* model_id,
const char* prompt,
uint32_t timeout_ms,
const XLLM_RequestParams* request_params);
/**
* @brief Generate chat completions from a conversation history
*
* Generates model responses for a multi-turn conversation using chat-formatted
* message history (user/assistant/system roles). Returns a dynamically
* allocated response struct that MUST be freed with xllm_llm_free_response().
*
* If request_params is NULL, this function automatically uses the default
* values from XLLM_LLM_REQUEST_PARAMS_DEFAULT (via
* xllm_llm_request_params_default()).
*
* @param handler Valid, initialized LLM instance handle (must not be NULL)
* @param model_id Null-terminated string of the loaded model ID
* @param messages Array of XLLM_ChatMessage structs (conversation history)
* @param messages_count Number of messages in the messages array (must be ≥ 0)
* @param timeout_ms Timeout in milliseconds (0 = no timeout)
* @param request_params Generation parameters (NULL = use defaults)
*
* @return Pointer to XLLM_Response on success; NULL ONLY if memory allocation
* fails (response->status indicates the actual result status)
*
* @response_status_codes
* - kSuccess: Valid chat response generated (check
* response->choices[0].message)
* - kNotInitialized: Handler not initialized
* - kInvalidRequest: Invalid messages (NULL with count>0, empty role/content)
* - kTimeout: Generation exceeded timeout_ms
*
* @warning Mandatory: Call xllm_llm_free_response() to release response memory
* @see xllm_llm_request_params_default, XLLM_LLM_REQUEST_PARAMS_DEFAULT,
* xllm_llm_free_response
*/
XLLM_CAPI_EXPORT XLLM_Response* xllm_llm_chat_completions(
XLLM_LLM_Handler* handler,
const char* model_id,
const XLLM_ChatMessage* messages,
size_t messages_count,
uint32_t timeout_ms,
const XLLM_RequestParams* request_params);
/**
* @brief Free all dynamically allocated memory in an XLLM_Response
*
* Releases all heap memory used by the response struct
*
* After freeing, all fields are reset to safe defaults (NULL/0) to prevent
* use-after-free.
*
* @param resp Pointer to XLLM_Response to free (NULL = no operation)
*
* @note Idempotent: Safe to call multiple times on the same response
* @warning Mandatory: Must be called after using completions/chat completions
* responses
* @see xllm_llm_completions, xllm_llm_chat_completions
*/
XLLM_CAPI_EXPORT void xllm_llm_free_response(XLLM_Response* resp);
#ifdef __cplusplus
}
#endif
#endif // XLLM_LLM_API_H

339
upstream_ref/xllm/xllm/c_api/rec.h Normal file
View File

@@ -0,0 +1,339 @@
/* Copyright 2025 The xLLM Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
https://github.com/jd-opensource/xllm/blob/main/LICENSE
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef XLLM_REC_API_H
#define XLLM_REC_API_H
#ifdef __cplusplus
extern "C" {
#endif
#include <stdbool.h>
#include <stddef.h>
#include <stdint.h>
#include "types.h"
/**
* @brief Opaque handle to a Generative Recommendation (REC) inference instance
* This handle encapsulates all internal state of a REC-specialized runtime,
* including:
* - Generative recommendation model weights (item embedding, ranking head)
* - Device context (CUDA/NPU streams for batch inference)
* - Generation cache (user behavior context, item candidate pool)
* - Runtime config (recommendation-specific decoding strategy)
* The handle MUST be created via xllm_rec_create() and destroyed via
* xllm_rec_destroy() to prevent memory/device resource leaks.
*/
typedef struct XLLM_REC_Handler XLLM_REC_Handler;
/**
* @brief Create a new Generative Recommendation (REC) inference instance handle
* This is the first function that must be called before using any other REC
* APIs.
* @return Valid XLLM_REC_Handler* on success; NULL if memory allocation fails
* @see xllm_rec_destroy
*/
XLLM_CAPI_EXPORT XLLM_REC_Handler* xllm_rec_create(void);
/**
* @brief Destroy a Generative Recommendation (REC) inference instance handle
* and release resources Frees all memory allocated for the REC instance,
* including:
* - Model weights (host/device memory for item embedding and ranking head)
* - Runtime context (CUDA/NPU streams, compute graphs for batch recommendation)
* - Generation cache (user behavior sequence, item candidate pool, attention
* cache)
* - Device resources (contexts, queues, memory pools for batch inference)
* This function is idempotent—calling with NULL has no effect.
* @param handler REC inference instance handle (NULL = no operation)
* @note Mandatory: Must be called to avoid memory/device resource leaks
* @see xllm_rec_create
*/
XLLM_CAPI_EXPORT void xllm_rec_destroy(XLLM_REC_Handler* handler);
/**
* @brief Helper to initialize XLLM_InitOptions with REC default values
* Copies the predefined XLLM_INIT_REC_OPTIONS_DEFAULT values into the target
* init_options struct. Convenient alternative to manually setting each field,
* ensuring consistency with REC best practices.
* @param init_options Pointer to XLLM_InitOptions to initialize (NULL = no-op)
* @see XLLM_INIT_REC_OPTIONS_DEFAULT, xllm_rec_initialize
*/
XLLM_CAPI_EXPORT void xllm_rec_init_options_default(
XLLM_InitOptions* init_options);
/**
* @brief Initialize the Generative Recommendation (REC) model and runtime
* environment Loads generative recommendation model weights from the specified
* path, configures target devices, initializes compute contexts, and prepares
* the recommendation inference runtime
* @param handler Valid REC inference instance handle (must not be NULL)
* @param model_path Null-terminated string of the REC model directory/file path
* (supports .bin/.pth/.safetensors formats with ranking head)
* @param devices Null-terminated string specifying target devices (format:
* "npu:0,1" (specific NPUs), "cuda:0" (single GPU), "auto"
* (automatic selection))
* @param init_options Advanced initialization options (NULL = use REC defaults)
* @return true if initialization succeeds; false on failure (see failure causes
* below)
* @par Failure Causes
* - Invalid handler (NULL or already destroyed)
* - Invalid model_path (non-existent, corrupted, or missing ranking head
* weights)
* - Invalid devices string (malformed format or unavailable devices)
* - Model load error (mismatched REC model architecture or embedding table
* corruption)
* - Device initialization failure (out of memory, driver error, insufficient
* batch size)
* @see xllm_rec_init_options_default, XLLM_INIT_REC_OPTIONS_DEFAULT,
* xllm_rec_create
*/
XLLM_CAPI_EXPORT bool xllm_rec_initialize(XLLM_REC_Handler* handler,
const char* model_path,
const char* devices,
const XLLM_InitOptions* init_options);
/**
* @brief Helper to initialize XLLM_RequestParams with REC default values
* Copies the predefined XLLM_REC_REQUEST_PARAMS_DEFAULT values into the target
* request_params struct.
* @param request_params Pointer to XLLM_RequestParams to initialize (NULL =
* no-op)
* @see XLLM_REC_REQUEST_PARAMS_DEFAULT, xllm_rec_text_completions,
* xllm_rec_token_completions, xllm_rec_chat_completions
*/
XLLM_CAPI_EXPORT void xllm_rec_request_params_default(
XLLM_RequestParams* request_params);
/**
* @brief Generate generative recommendation text completions for a user prompt
* Generates recommendation-focused continuation text for the input user prompt
* using the initialized REC model
* @param handler Valid, initialized REC inference instance handle (must not be
* NULL)
* @param model_id Null-terminated string of the loaded REC model ID (must match
* model_path)
* @param prompt Null-terminated string of user input prompt (non-empty,
* recommendation-focused)
* @param timeout_ms Timeout in milliseconds (0 = no timeout, wait indefinitely)
* @param request_params Generation parameters (NULL = use REC defaults)
* @return Pointer to XLLM_Response on success; NULL ONLY if memory allocation
* fails (response->status indicates the actual result status)
* @par Response Status Codes
* - kSuccess: Valid recommendation response generated (check response->choices
* for item list + explanations)
* - kNotInitialized: Handler not initialized with xllm_rec_initialize()
* - kInvalidRequest: Invalid prompt (empty/NULL) or model_id (mismatch)
* - kTimeout: Generation exceeded timeout_ms (partial recommendation results
* may be available)
* @warning Mandatory: Call xllm_rec_free_response() to release response memory
* @see xllm_rec_request_params_default, XLLM_REC_REQUEST_PARAMS_DEFAULT,
* xllm_rec_free_response
*/
XLLM_CAPI_EXPORT XLLM_Response* xllm_rec_text_completions(
XLLM_REC_Handler* handler,
const char* model_id,
const char* prompt,
uint32_t timeout_ms,
const XLLM_RequestParams* request_params);
/**
* @brief Generate generative recommendation completions for tokenized input
* (TOKEN ID INPUT) Generates recommendation results from pre-tokenized user
* input (bypasses the REC model's tokenizer)
*
* @param handler Valid, initialized REC inference instance handle (must not be
* NULL) Created via xllm_rec_create() and initialized via xllm_rec_initialize()
* @param model_id Null-terminated string of the loaded REC model ID (must match
* the model_path used in xllm_rec_initialize())
* @param token_ids Pointer to int32_t array of pre-tokenized input IDs (NULL
* only if token_size = 0) Token IDs must be compatible with the REC model's
* tokenizer vocabulary (e.g., GPT-2/BERT token IDs for text-based REC models)
* @param token_size Number of tokens in the token_ids array (must be ≥ 0)
* Valid ranges: 1 ≤ token_size ≤ xxx (model-dependent max input
* length) token_size = 0 will return kInvalidRequest status
* @param timeout_ms Timeout in milliseconds (0 = no timeout, wait indefinitely)
* @param request_params Generation parameters (NULL = use REC defaults)
*
* @return Pointer to XLLM_Response on success; NULL ONLY if memory allocation
* fails (response->status indicates the actual result status, even if non-NULL)
*
* @par Response Status Codes (XLLM_StatusCode)
* - kSuccess: Valid recommendation response generated
* Check response->choices for recommended item list and explanation
* text
* - kNotInitialized: Handler not initialized with xllm_rec_initialize()
* - kModelNotFound: model_id does not match any loaded REC model
* - kInvalidRequest:
* - token_ids = NULL and token_size > 0 (invalid null pointer with non-zero
* size)
* - token_size = 0 (empty token input)
* - token_ids contain invalid IDs (out of vocabulary range)
* - model_id is NULL/empty/mismatch
* - kTimeout: Generation exceeded timeout_ms
* - kInternalError: Internal REC runtime error (e.g., token embedding failure,
* item retrieval error)
* @warning Mandatory: Call xllm_rec_free_response() to release response memory
* @note 1. Token IDs must be generated using the SAME tokenizer as the REC
* model (e.g., same vocab.txt)
* 2. Invalid token IDs (e.g., < 0 or > vocab_size) will trigger
* kInvalidRequest or kInternalError
* 3. For token_size > model's max input length, the input will be
* truncated to max length
* @see xllm_rec_request_params_default, XLLM_REC_REQUEST_PARAMS_DEFAULT,
* xllm_rec_free_response
*/
XLLM_CAPI_EXPORT XLLM_Response* xllm_rec_token_completions(
XLLM_REC_Handler* handler,
const char* model_id,
const int32_t* token_ids,
size_t token_size,
uint32_t timeout_ms,
const XLLM_RequestParams* request_params);
/**
* @brief Generate generative recommendation completions for multimodal input
* (TOKEN ID + MULTIMODAL DATA INPUT)
* @details Generates recommendation results from pre-tokenized text input
* (MANDATORY) supplemented with multimodal data that replaces/augments
* information for specific tokens in the token_ids array. This API extends
* xllm_rec_token_completions to support multi-modal recommendation scenarios
* where partial text tokens are enriched with image/audio/video/embedding
* features (e.g., replacing product text tokens with image embeddings).
*
* @param handler Valid, initialized REC inference instance handle (must not be
* NULL) Created via xllm_rec_create() and initialized via xllm_rec_initialize()
* @param model_id Null-terminated string of the loaded REC model ID (must match
* the model_path used in xllm_rec_initialize())
* Must be a multi-modal REC model (text-only models will return
* kInvalidRequest)
* @param token_ids Pointer to int32_t array of pre-tokenized text input IDs
* (MUST NOT be NULL) Token IDs must be compatible with the REC model's
* tokenizer vocabulary This is the core input and cannot be empty (token_size >
* 0 required)
* @param token_size Number of tokens in the token_ids array (MUST be ≥ 1)
* Valid ranges: 1 ≤ token_size ≤ model-dependent max input
* length token_size = 0 will return kInvalidRequest status (core text input
* required)
* @param mm_data Pointer to multi-modal data container (XLLM_MM_Data) (NULL =
* no multimodal augmentation) Used to replace/augment information for specific
* tokens in token_ids (via XLLM_MM_TokenPos) Supports
* image/audio/video/embedding modalities (see XLLM_MM_Type) Must be valid
* (mm_data->type_mask != XLLM_MM_TYPE_NONE) if non-NULL, and token positions in
* mm_data must be within [0, token_size-1] (out-of-range positions trigger
* kInvalidRequest)
* @param timeout_ms Timeout in milliseconds (0 = no timeout, wait indefinitely)
* @param request_params Generation parameters (NULL = use REC defaults)
* See XLLM_RequestParams for configurable options (e.g.,
* top_k, top_p)
* @return Pointer to XLLM_Response on success; NULL ONLY if memory allocation
* fails (response->status indicates the actual result status, even if
* non-NULL)
* @par Response Status Codes (XLLM_StatusCode)
* - kSuccess: Valid multi-modal recommendation response generated
* Check response->choices for recommended item list and explanation
* text Multimodal data has been applied to augment/replace specified tokens
* - kNotInitialized: Handler not initialized with xllm_rec_initialize()
* - kModelNotFound: model_id does not match any loaded REC model
* - kInvalidRequest:
* - token_ids = NULL (core text input is mandatory)
* - token_size = 0 (empty core text input)
* - token_ids contain invalid IDs (out of vocabulary range)
* - model_id is NULL/empty/mismatch or is a text-only model
* - mm_data is non-NULL but invalid:
* - mm_data->type_mask = XLLM_MM_TYPE_NONE (empty multimodal data)
* - token positions in mm_data (XLLM_MM_TokenPos) are out of [0,
* token_size-1] range
* - mismatched tensor types/shape in mm_data (e.g., embedding dim mismatch)
* - kTimeout: Generation exceeded timeout_ms
* - kInternalError: Internal REC runtime error (e.g., multimodal embedding
* fusion failure, token augmentation/replacement error, item retrieval error)
* @warning Mandatory: Call xllm_rec_free_response() to release response memory
* Failing to free will cause memory leaks
* @note 1. Token IDs must be generated using the SAME tokenizer as the REC
* model (e.g., same vocab.txt)
* 2. Invalid token IDs (e.g., < 0 or > vocab_size) will trigger
* kInvalidRequest or kInternalError
* 3. For token_size > model's max input length, the input will be
* truncated to max length
* 4. mm_data is used to replace/augment specific tokens (via
* XLLM_MM_TokenPos.offset/length):
* - offset: start index of tokens in token_ids to be
* augmented/replaced
* - length: number of consecutive tokens to apply multimodal data to
* 5. If mm_data is NULL, this API behaves identically to
* xllm_rec_token_completions (text-only inference)
* 6. Multimodal data must be aligned with token positions (offset +
* length ≤ token_size)
* @see xllm_rec_token_completions, xllm_rec_request_params_default,
* XLLM_REC_REQUEST_PARAMS_DEFAULT, xllm_rec_free_response, XLLM_MM_Data,
* XLLM_MM_TokenPos
*/
XLLM_CAPI_EXPORT XLLM_Response* xllm_rec_multimodal_completions(
XLLM_REC_Handler* handler,
const char* model_id,
const int32_t* token_ids,
size_t token_size,
const XLLM_MM_Data* mm_data,
uint32_t timeout_ms,
const XLLM_RequestParams* request_params);
/**
* @brief Generate generative recommendation chat completions from multi-turn
* conversation history Generates personalized recommendation responses for a
* multi-turn user-assistant conversation
* @param handler Valid, initialized REC inference instance handle (must not be
* NULL)
* @param model_id Null-terminated string of the loaded REC model ID
* @param messages Array of XLLM_ChatMessage structs (recommendation-focused
* conversation history)
* @param messages_count Number of messages in the messages array (must be ≥ 0)
* @param timeout_ms Timeout in milliseconds (0 = no timeout, wait indefinitely)
* @param request_params Generation parameters (NULL = use REC defaults)
* @return Pointer to XLLM_Response on success; NULL ONLY if memory allocation
* fails (response->status indicates the actual result status)
* @par Response Status Codes
* - kSuccess: Valid chat recommendation response generated (check
* response->choices[0].message for item list)
* - kNotInitialized: Handler not initialized with xllm_rec_initialize()
* - kInvalidRequest: Invalid messages (NULL with count>0, empty role/content,
* non-recommendation context)
* - kTimeout: Generation exceeded timeout_ms
* @warning Mandatory: Call xllm_rec_free_response() to release response memory
* @see xllm_rec_request_params_default, XLLM_REC_REQUEST_PARAMS_DEFAULT,
* xllm_rec_free_response
*/
XLLM_CAPI_EXPORT XLLM_Response* xllm_rec_chat_completions(
XLLM_REC_Handler* handler,
const char* model_id,
const XLLM_ChatMessage* messages,
size_t messages_count,
uint32_t timeout_ms,
const XLLM_RequestParams* request_params);
/**
* @brief Free all dynamically allocated memory in a generative recommendation
* XLLM_Response Releases all heap memory used by the REC response struct
* @param resp Pointer to XLLM_Response to free (NULL = no operation)
* @warning Mandatory: Must be called after using REC completions/chat
* completions responses
* @see xllm_rec_text_completions, xllm_rec_token_completions,
* xllm_rec_chat_completions
*/
XLLM_CAPI_EXPORT void xllm_rec_free_response(XLLM_Response* resp);
#ifdef __cplusplus
}
#endif
#endif // XLLM_REC_API_H

280
upstream_ref/xllm/xllm/c_api/test/CMakeLists.txt Normal file
View File

@@ -0,0 +1,280 @@
cmake_minimum_required(VERSION 3.10)
project(xllm_capi_test LANGUAGES CXX)
set(CMAKE_CXX_STANDARD 14)
set(CMAKE_CXX_STANDARD_REQUIRED ON)
# -----------------------------------------------------------------------------
# Dependencies — Protobuf (vcpkg uses protobuf-config.cmake; package name is "protobuf")
# -----------------------------------------------------------------------------
# Recommended (matches main project / vcpkg.json):
# cmake -B build \
# -DCMAKE_TOOLCHAIN_FILE=$VCPKG_ROOT/scripts/buildsystems/vcpkg.cmake \
# -DVCPKG_MANIFEST_DIR=<path/to/xllm/repo_root>
# To reuse an existing xLLM build without re-installing deps:
# -DCMAKE_PREFIX_PATH=<xllm_build>/vcpkg_installed/<triplet>
#
# Fallback: system FindProtobuf / pkg-config / manual paths (no vcpkg).
find_package(protobuf CONFIG QUIET)
if(NOT protobuf_FOUND)
find_package(Protobuf CONFIG QUIET)
endif()
# vcpkg protobuf::protoc may not set Protobuf_PROTOC_EXECUTABLE; resolve from target.
if((protobuf_FOUND OR Protobuf_FOUND) AND TARGET protobuf::libprotobuf)
if(NOT Protobuf_PROTOC_EXECUTABLE AND TARGET protobuf::protoc)
get_target_property(Protobuf_PROTOC_EXECUTABLE protobuf::protoc IMPORTED_LOCATION)
if(NOT Protobuf_PROTOC_EXECUTABLE)
get_target_property(Protobuf_PROTOC_EXECUTABLE protobuf::protoc IMPORTED_LOCATION_RELEASE)
endif()
if(NOT Protobuf_PROTOC_EXECUTABLE)
get_target_property(Protobuf_PROTOC_EXECUTABLE protobuf::protoc IMPORTED_LOCATION_DEBUG)
endif()
endif()
if(NOT Protobuf_PROTOC_EXECUTABLE)
message(FATAL_ERROR
"Protobuf CONFIG found but could not resolve protoc (protobuf::protoc / Protobuf_PROTOC_EXECUTABLE).")
endif()
message(STATUS "Found Protobuf (CONFIG / vcpkg): protoc=${Protobuf_PROTOC_EXECUTABLE}")
set(_PROTOBUF_LIB protobuf::libprotobuf)
else()
set(Protobuf_FOUND FALSE)
set(protobuf_FOUND FALSE)
find_package(Protobuf QUIET)
if(Protobuf_FOUND)
message(STATUS "Found Protobuf (module): protoc=${Protobuf_PROTOC_EXECUTABLE}")
endif()
if(NOT Protobuf_FOUND)
find_package(PkgConfig QUIET)
if(PkgConfig_FOUND)
pkg_check_modules(PC_PROTOBUF QUIET protobuf)
endif()
if(PC_PROTOBUF_FOUND)
set(Protobuf_INCLUDE_DIRS ${PC_PROTOBUF_INCLUDE_DIRS})
set(Protobuf_LIBRARIES ${PC_PROTOBUF_LIBRARIES})
find_program(Protobuf_PROTOC_EXECUTABLE protoc)
if(NOT Protobuf_PROTOC_EXECUTABLE)
message(FATAL_ERROR
"Found libprotobuf via pkg-config, but protoc was not found in PATH. "
"Please install protoc or set Protobuf_PROTOC_EXECUTABLE.")
endif()
set(_PROTOBUF_LIB ${Protobuf_LIBRARIES})
endif()
endif()
if(NOT Protobuf_FOUND AND NOT PC_PROTOBUF_FOUND)
find_path(Protobuf_INCLUDE_DIR
NAMES google/protobuf/message.h
PATHS /usr/include /usr/local/include
)
find_library(Protobuf_LIBRARY
NAMES protobuf libprotobuf
PATHS /usr/lib /usr/lib64 /usr/local/lib /usr/local/lib64
)
find_program(Protobuf_PROTOC_EXECUTABLE
NAMES protoc
PATHS /usr/bin /usr/local/bin
)
if(Protobuf_INCLUDE_DIR AND Protobuf_LIBRARY AND Protobuf_PROTOC_EXECUTABLE)
set(Protobuf_INCLUDE_DIRS ${Protobuf_INCLUDE_DIR})
set(Protobuf_LIBRARIES ${Protobuf_LIBRARY})
set(_PROTOBUF_LIB ${Protobuf_LIBRARY})
message(STATUS "Found Protobuf manually:"
" include=${Protobuf_INCLUDE_DIR}"
" lib=${Protobuf_LIBRARY}"
" protoc=${Protobuf_PROTOC_EXECUTABLE}")
else()
message(FATAL_ERROR
"Could NOT find Protobuf.\n"
"Preferred: configure with vcpkg like the main xLLM build, e.g.\n"
" -DCMAKE_TOOLCHAIN_FILE=<vcpkg>/scripts/buildsystems/vcpkg.cmake\n"
" -DVCPKG_MANIFEST_DIR=<xllm_repo_root>\n"
"Or install system protobuf + protoc, or set:\n"
" -DProtobuf_INCLUDE_DIR=/path/to/include\n"
" -DProtobuf_LIBRARY=/path/to/libprotobuf.so\n"
" -DProtobuf_PROTOC_EXECUTABLE=/path/to/protoc\n")
endif()
endif()
if(TARGET protobuf::libprotobuf)
set(_PROTOBUF_LIB protobuf::libprotobuf)
endif()
endif()
# -----------------------------------------------------------------------------
# Generate protobuf sources from xllm_test.proto
# -----------------------------------------------------------------------------
set(XLLM_TEST_PROTO ${CMAKE_CURRENT_LIST_DIR}/xllm_test.proto)
set(XLLM_TEST_PB_CC ${CMAKE_CURRENT_BINARY_DIR}/xllm_test.pb.cc)
set(XLLM_TEST_PB_H ${CMAKE_CURRENT_BINARY_DIR}/xllm_test.pb.h)
add_custom_command(
OUTPUT ${XLLM_TEST_PB_CC} ${XLLM_TEST_PB_H}
COMMAND ${Protobuf_PROTOC_EXECUTABLE}
--cpp_out=${CMAKE_CURRENT_BINARY_DIR}
-I ${CMAKE_CURRENT_LIST_DIR}
${XLLM_TEST_PROTO}
DEPENDS ${XLLM_TEST_PROTO}
COMMENT "Generating xllm_test.pb.cc/h from xllm_test.proto"
VERBATIM
)
add_library(c_api_test_proto STATIC ${XLLM_TEST_PB_CC})
target_include_directories(c_api_test_proto PUBLIC ${CMAKE_CURRENT_BINARY_DIR})
if(TARGET protobuf::libprotobuf)
target_include_directories(c_api_test_proto PUBLIC
$<TARGET_PROPERTY:protobuf::libprotobuf,INTERFACE_INCLUDE_DIRECTORIES>)
elseif(Protobuf_INCLUDE_DIRS)
target_include_directories(c_api_test_proto PUBLIC ${Protobuf_INCLUDE_DIRS})
endif()
# PRIVATE: xllm_test controls link order (brpc before protobuf) for static archives.
target_link_libraries(c_api_test_proto PRIVATE ${_PROTOBUF_LIB})
# -----------------------------------------------------------------------------
# brpc + gflags + glog (server binary; brpc uses glog — link glog after libbrpc.a)
# -----------------------------------------------------------------------------
find_package(gflags CONFIG REQUIRED)
find_package(glog CONFIG REQUIRED)
find_package(Threads REQUIRED)
find_package(OpenSSL REQUIRED)
# brpc: main xLLM may use <repo>/build/third_party/brpc/output or
# <repo>/build/<toolchain>/third_party/brpc/output (e.g. cmake.linux-aarch64-*).
# Headers: output/include next to the library, else third_party/brpc/src.
get_filename_component(_XLLM_REPO_ROOT "${CMAKE_CURRENT_LIST_DIR}/../../.."
ABSOLUTE)
if(NOT BRPC_ROOT)
if(DEFINED ENV{BRPC_ROOT} AND NOT "$ENV{BRPC_ROOT}" STREQUAL "")
set(BRPC_ROOT "$ENV{BRPC_ROOT}")
else()
set(BRPC_ROOT
"${_XLLM_REPO_ROOT}/build/third_party/brpc/output")
endif()
endif()
set(BRPC_ROOT "${BRPC_ROOT}" CACHE PATH
"brpc prefix: include/brpc/server.h and lib/libbrpc.a (override if needed)")
set(BRPC_LIB "")
foreach(_brpc_lib_cand
"${BRPC_ROOT}/lib/libbrpc.a"
"${BRPC_ROOT}/lib/libbrpc.so"
"${_XLLM_REPO_ROOT}/build/third_party/brpc/output/lib/libbrpc.a"
"${_XLLM_REPO_ROOT}/build/third_party/brpc/output/lib/libbrpc.so")
if(EXISTS "${_brpc_lib_cand}")
set(BRPC_LIB "${_brpc_lib_cand}")
break()
endif()
endforeach()
# e.g. build/cmake.linux-aarch64-cpython-311/third_party/brpc/output/lib/...
if(NOT BRPC_LIB)
file(GLOB _brpc_lib_glob LIST_DIRECTORIES false
"${_XLLM_REPO_ROOT}/build/*/third_party/brpc/output/lib/libbrpc.a"
"${_XLLM_REPO_ROOT}/build/*/third_party/brpc/output/lib/libbrpc.so")
if(_brpc_lib_glob)
list(SORT _brpc_lib_glob)
list(GET _brpc_lib_glob 0 BRPC_LIB)
endif()
endif()
if(NOT BRPC_LIB)
find_library(BRPC_LIB NAMES brpc libbrpc.a
PATHS "${BRPC_ROOT}/lib"
"${CMAKE_PREFIX_PATH}/lib"
/usr/local/lib
/usr/lib64
/usr/lib)
endif()
set(BRPC_INCLUDE_DIR "")
if(BRPC_LIB)
get_filename_component(_brpc_out "${BRPC_LIB}" DIRECTORY)
get_filename_component(_brpc_out "${_brpc_out}" DIRECTORY)
if(EXISTS "${_brpc_out}/include/brpc/server.h")
set(BRPC_INCLUDE_DIR "${_brpc_out}/include")
endif()
endif()
if(NOT BRPC_INCLUDE_DIR AND EXISTS "${BRPC_ROOT}/include/brpc/server.h")
set(BRPC_INCLUDE_DIR "${BRPC_ROOT}/include")
endif()
if(NOT BRPC_INCLUDE_DIR
AND EXISTS "${_XLLM_REPO_ROOT}/third_party/brpc/src/brpc/server.h")
set(BRPC_INCLUDE_DIR "${_XLLM_REPO_ROOT}/third_party/brpc/src")
endif()
if(NOT BRPC_INCLUDE_DIR)
message(FATAL_ERROR
"brpc headers not found (BRPC_ROOT='${BRPC_ROOT}').\n"
" Expected include/brpc/server.h next to libbrpc, or submodule\n"
" '${_XLLM_REPO_ROOT}/third_party/brpc/src/brpc/server.h'.\n"
" Set -DBRPC_ROOT=... to a prefix with include/brpc/server.h, or init\n"
" git submodules so third_party/brpc exists.")
endif()
if(NOT BRPC_LIB)
message(FATAL_ERROR
"libbrpc not found (BRPC_ROOT='${BRPC_ROOT}').\n"
" xllm_test links brpc from the main xLLM build, e.g.\n"
" ${_XLLM_REPO_ROOT}/build/third_party/brpc/output/lib/libbrpc.a\n"
" ${_XLLM_REPO_ROOT}/build/*/third_party/brpc/output/lib/libbrpc.a\n"
" Build the main project first, or pass -DBRPC_ROOT=/path/to/.../output\n"
" (directory with include/ and lib/libbrpc.a).")
endif()
find_package(leveldb CONFIG REQUIRED)
find_package(ZLIB REQUIRED)
# -----------------------------------------------------------------------------
# Build xllm_test (links external /usr/local/xllm/lib/libxllm.so)
# -----------------------------------------------------------------------------
add_executable(xllm_test
${CMAKE_CURRENT_LIST_DIR}/xllm_test.cpp
${CMAKE_CURRENT_LIST_DIR}/utils.cpp
)
# rec.h / types.h: installed C API under /usr/local/xllm/include (run xllm/c_api/install.sh).
# Local test headers (utils.h): ${CMAKE_CURRENT_LIST_DIR}
target_include_directories(xllm_test
PRIVATE
${CMAKE_CURRENT_BINARY_DIR}
/usr/local/xllm/include
${CMAKE_CURRENT_LIST_DIR}
${BRPC_INCLUDE_DIR}
)
target_link_directories(xllm_test PRIVATE /usr/local/xllm/lib)
# Static libbrpc.a pulls protobuf gzip + glog symbols; libprotobuf must appear
# after brpc (or use --start-group) so GzipOutputStream etc. resolve; ZLIB for gzip.
# --start-group/--end-group: GNU ld needs them so libprotobuf resolves symbols
# referenced from libbrpc.a (e.g. GzipOutputStream) when linking static archives.
if(CMAKE_CXX_COMPILER_ID MATCHES "GNU|Clang" AND NOT MSVC)
set(_BRPC_LINK_GROUP_START -Wl,--start-group)
set(_BRPC_LINK_GROUP_END -Wl,--end-group)
else()
set(_BRPC_LINK_GROUP_START "")
set(_BRPC_LINK_GROUP_END "")
endif()
target_link_libraries(xllm_test
PRIVATE
c_api_test_proto
gflags::gflags
${_BRPC_LINK_GROUP_START}
${BRPC_LIB}
${_PROTOBUF_LIB}
glog::glog
ZLIB::ZLIB
${_BRPC_LINK_GROUP_END}
leveldb::leveldb
OpenSSL::SSL
OpenSSL::Crypto
Threads::Threads
dl
xllm
)
# Keep runtime able to locate libxllm.so without setting LD_LIBRARY_PATH.
set_target_properties(xllm_test PROPERTIES
BUILD_RPATH "/usr/local/xllm/lib"
INSTALL_RPATH "/usr/local/xllm/lib"
)

109
upstream_ref/xllm/xllm/c_api/test/README.md Normal file
View File

@@ -0,0 +1,109 @@
# c_api/test — `xllm_test` 说明
## 作用
`xllm_test` 是一个基于 **brpc** 的小型服务进程，用于在 **RPC 层** 验证 xLLM 的 **C API**（`xllm/c_api/llm.h` 或 `rec.h`）：
- 对客户端暴露 **一个** RPC：`Inference(XLLM_Request) -> XLLM_Response`（定义见 `xllm_test.proto`）。
- 根据请求里的 **`call_function`** 字符串，转发到对应的 C API（例如 `xllm_llm_completions`、`xllm_rec_text_completions` 等）。
- 请求/响应中的结构与 `types.h` 对齐，由 `utils.cpp` 在 **Protobuf** 与 **C 结构体** 之间做转换。
**注意**：一次进程只加载 **一种** 后端，由 **`--backend`** 决定：
| `--backend` | 使用的 C API | 仅有效的 `call_function` 前缀 |
|-------------|--------------|--------------------------------|
| `llm` | `llm.h` | `xllm_llm_*` |
| `rec` | `rec.h` | `xllm_rec_*` |
若后端与 `call_function` 不匹配（例如在 `rec` 模式下调用 `xllm_llm_completions`），会返回错误（例如 handler 为空）。
---
## 依赖与前置条件
1. **已安装的 C API 头文件与 `libxllm.so`**
默认按 **`/usr/local/xllm/include`** 与 **`/usr/local/xllm/lib`** 查找（与 `CMakeLists.txt` 一致）。
若尚未安装，可在仓库内执行 `xllm/c_api/install.sh`（或你们环境约定的安装方式）。
2. **主工程已构建出的 brpc**
`libbrpc.a`（及头文件）通常位于仓库根目录下类似路径：
`build/third_party/brpc/output/` 或
`build/<toolchain>/third_party/brpc/output/`（例如 `cmake.linux-aarch64-cpython-311`）。
CMake 会自动在 `build/*/third_party/brpc/output` 下搜索；若仍找不到，可设置：
`-DBRPC_ROOT=/path/to/.../third_party/brpc/output` 或环境变量 **`BRPC_ROOT`**。
3. **Protobuf、gflags、glog、leveldb、OpenSSL、Zlib**
推荐与主工程一致，使用 **vcpkg**（仓库根目录 `vcpkg.json`），配置时传入
`-DCMAKE_TOOLCHAIN_FILE=$VCPKG_ROOT/scripts/buildsystems/vcpkg.cmake`、
`-DVCPKG_MANIFEST_DIR=<xllm 仓库根目录>`。
---
## 编译
在 **`xllm/c_api/test`** 目录下新建构建目录并配置、编译（请将占位路径换成你本机路径）：
```bash
cd xllm/c_api/test
cmake -B build \
-DCMAKE_BUILD_TYPE=Release \
-DCMAKE_TOOLCHAIN_FILE=$VCPKG_ROOT/scripts/buildsystems/vcpkg.cmake \
-DVCPKG_MANIFEST_DIR=/path/to/xllm
cmake --build build -j$(nproc)
```
生成可执行文件：**`build/xllm_test`**（具体路径以 CMake 生成位置为准）。
若 vcpkg 依赖已安装在主工程构建目录中，也可通过 **`CMAKE_PREFIX_PATH`** 指向
`<主工程 build>/vcpkg_installed/<triplet>`，避免重复安装。
---
## 运行
1. 编辑示例 flags：**`xllm_test.flags`**（至少设置 **`--model_path`**、**`--devices`**，并按需设置 **`--backend=llm`** 或 **`--backend=rec`**）。
2. 启动服务：
```bash
/path/to/build/xllm_test --flagfile=/path/to/xllm/c_api/test/xllm_test.flags
```
或在命令行直接传参，例如：
```bash
./build/xllm_test \
--backend=rec \
--model_path=/path/to/model \
--devices=auto \
--port=8000
```
3. **监听地址**
- 默认使用 **`--port`**（如 `8000`）在 `0.0.0.0` 上监听。
- 若设置 **`--listen_addr=host:port`**，则优先使用该地址（与 `xllm_test.flags` 中注释一致）。
4. **调用方式**
任意支持 **brpc + 同一套 `xllm_test.proto`** 的客户端，向上述地址发起 **`XllmRecCapiService/Inference`**，在 **`XLLM_Request.call_function`** 中填入与当前 **`--backend`** 一致的 API 名称即可。
---
## 目录内主要文件
| 文件 | 说明 |
|------|------|
| `xllm_test.cpp` | brpc 服务入口、`Inference` 分发逻辑 |
| `xllm_test.proto` | RPC 与消息定义 |
| `utils.cpp` / `utils.h` | Protobuf ↔ C API 类型转换、gflags 定义 |
| `xllm_test.flags` | 示例运行参数 |
| `CMakeLists.txt` | 构建配置 |
---
## 常见问题
- **`brpc` / `libbrpc.a` 找不到**：先在仓库根目录完整配置并编译主工程，使 `third_party/brpc` 产物出现；或使用 `-DBRPC_ROOT`。
- **链接或运行找不到 `libxllm.so`**：确认已安装到 **`/usr/local/xllm/lib`**，或自行修改 `CMakeLists.txt` 中的 include/lib 路径并设置 **`LD_LIBRARY_PATH`**。
- **与主进程 `127.0.0.1:18899` 相关日志**：那是 xLLM **分布式 engine/worker** 的地址，与 `xllm_test` 的 **`--port` / `--listen_addr`** 无关；需按主工程文档单独启动 engine。

647
upstream_ref/xllm/xllm/c_api/test/utils.cpp Normal file
View File

@@ -0,0 +1,647 @@
/* Copyright 2026 The xLLM Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
https://github.com/jd-opensource/xllm/blob/main/LICENSE
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#include "utils.h"
#include <gflags/gflags.h>
#include <algorithm>
#include <cstring>
#include <string>
// --- Server + XLLM_InitOptions gflags (defaults aligned with REC defaults) ---
DEFINE_string(model_path, "", "Path to REC model weights");
DEFINE_string(devices, "auto", "Devices string, e.g. npu:0 or auto");
DEFINE_int32(port, 8000, "brpc TCP port");
DEFINE_string(listen_addr,
"",
"If non-empty, brpc listen endpoint (host:port), overrides port");
DEFINE_int32(idle_timeout_s,
-1,
"brpc connection idle timeout in seconds; -1 = no limit");
DEFINE_string(backend,
"rec",
"C API mode for xllm_test: \"llm\" (c_api/llm.h) or \"rec\" "
"(c_api/rec.h); only one is loaded");
DEFINE_bool(enable_chunked_prefill, false, "");
DEFINE_bool(enable_prefill_sp, false, "");
DEFINE_bool(enable_prefix_cache, false, "");
DEFINE_bool(enable_disagg_pd, false, "");
DEFINE_bool(enable_pd_ooc, false, "");
DEFINE_bool(enable_schedule_overlap, false, "");
DEFINE_bool(enable_shm, false, "");
DEFINE_uint32(transfer_listen_port, 26000, "");
DEFINE_uint32(nnodes, 1, "");
DEFINE_uint32(node_rank, 0, "");
DEFINE_uint32(dp_size, 1, "");
DEFINE_uint32(ep_size, 1, "");
DEFINE_uint32(block_size, 1, "");
DEFINE_uint32(max_cache_size, 1000000, "");
DEFINE_uint32(max_tokens_per_batch, 4096, "");
DEFINE_uint32(max_seqs_per_batch, 4, "");
DEFINE_uint32(max_tokens_per_chunk_for_prefill, 0, "");
DEFINE_uint32(num_speculative_tokens, 0, "");
DEFINE_uint32(num_request_handling_threads, 4, "");
DEFINE_uint32(expert_parallel_degree, 0, "");
DEFINE_uint32(server_idx, 0, "");
DEFINE_uint32(beam_width, 128, "");
DEFINE_uint32(max_decode_rounds, 3, "");
DEFINE_uint32(max_token_per_req, 1000, "");
DEFINE_double(max_memory_utilization, 0.55, "");
DEFINE_string(init_task, "generate", "XLLM_InitOptions.task");
DEFINE_string(communication_backend, "lccl", "");
DEFINE_string(instance_role, "DEFAULT", "");
DEFINE_string(device_ip, "", "");
DEFINE_string(master_node_addr, "127.0.0.1:18899", "");
DEFINE_string(xservice_addr, "", "");
DEFINE_string(instance_name, "", "");
DEFINE_string(kv_cache_transfer_mode, "PUSH", "");
// Not named "log_dir": glog already registers FLAGS_log_dir.
DEFINE_string(xllm_init_log_dir, "", "");
DEFINE_string(draft_model, "", "");
DEFINE_string(draft_devices, "", "");
namespace xllm_capi_test {
namespace {
void CopyToFixed(char* dst, const std::string& s, size_t cap) {
if (cap == 0) {
return;
}
std::strncpy(dst, s.c_str(), cap - 1);
dst[cap - 1] = '\0';
}
std::unique_ptr<char[]> CopyCStr(const std::string& s) {
auto p = std::make_unique<char[]>(s.size() + 1);
if (!s.empty()) {
std::memcpy(p.get(), s.data(), s.size());
}
p[s.size()] = '\0';
return p;
}
size_t TensorNumElements(const XLLM_Dims& d) {
if (d.rank <= 0) {
return 0;
}
size_t n = 1;
for (int i = 0; i < d.rank && i < 8; ++i) {
if (d.dim[i] <= 0) {
return 0;
}
n *= static_cast<size_t>(d.dim[i]);
}
return n;
}
size_t DTypeSize(XLLM_DataType dt) {
switch (dt) {
case XLLM_DTYPE_FLOAT16:
case XLLM_DTYPE_BFLOAT16:
return 2;
case XLLM_DTYPE_FLOAT32:
return 4;
case XLLM_DTYPE_FLOAT64:
return 8;
case XLLM_DTYPE_INT8:
case XLLM_DTYPE_UINT8:
return 1;
case XLLM_DTYPE_INT16:
case XLLM_DTYPE_UINT16:
return 2;
case XLLM_DTYPE_INT32:
case XLLM_DTYPE_UINT32:
return 4;
case XLLM_DTYPE_INT64:
case XLLM_DTYPE_UINT64:
return 8;
case XLLM_DTYPE_BOOL:
return 1;
case XLLM_DTYPE_STRING:
case XLLM_DTYPE_UNDEFINED:
default:
return 0;
}
}
} // namespace
void ApplyGflagsToXllmInitOptions(XLLM_InitOptions* o) {
o->enable_chunked_prefill = FLAGS_enable_chunked_prefill;
o->enable_prefill_sp = FLAGS_enable_prefill_sp;
o->enable_prefix_cache = FLAGS_enable_prefix_cache;
o->enable_disagg_pd = FLAGS_enable_disagg_pd;
o->enable_pd_ooc = FLAGS_enable_pd_ooc;
o->enable_schedule_overlap = FLAGS_enable_schedule_overlap;
o->enable_shm = FLAGS_enable_shm;
o->transfer_listen_port = FLAGS_transfer_listen_port;
o->nnodes = FLAGS_nnodes;
o->node_rank = FLAGS_node_rank;
o->dp_size = FLAGS_dp_size;
o->ep_size = FLAGS_ep_size;
o->block_size = FLAGS_block_size;
o->max_cache_size = FLAGS_max_cache_size;
o->max_tokens_per_batch = FLAGS_max_tokens_per_batch;
o->max_seqs_per_batch = FLAGS_max_seqs_per_batch;
o->max_tokens_per_chunk_for_prefill = FLAGS_max_tokens_per_chunk_for_prefill;
o->num_speculative_tokens = FLAGS_num_speculative_tokens;
o->num_request_handling_threads = FLAGS_num_request_handling_threads;
o->expert_parallel_degree = FLAGS_expert_parallel_degree;
o->server_idx = FLAGS_server_idx;
o->beam_width = FLAGS_beam_width;
o->max_decode_rounds = FLAGS_max_decode_rounds;
o->max_token_per_req = FLAGS_max_token_per_req;
o->max_memory_utilization = static_cast<float>(FLAGS_max_memory_utilization);
CopyToFixed(o->task, FLAGS_init_task, XLLM_META_STRING_FIELD_MAX_LEN);
CopyToFixed(o->communication_backend,
FLAGS_communication_backend,
XLLM_META_STRING_FIELD_MAX_LEN);
CopyToFixed(
o->instance_role, FLAGS_instance_role, XLLM_META_STRING_FIELD_MAX_LEN);
CopyToFixed(o->device_ip, FLAGS_device_ip, XLLM_META_STRING_FIELD_MAX_LEN);
CopyToFixed(o->master_node_addr,
FLAGS_master_node_addr,
XLLM_META_STRING_FIELD_MAX_LEN);
CopyToFixed(
o->xservice_addr, FLAGS_xservice_addr, XLLM_META_STRING_FIELD_MAX_LEN);
CopyToFixed(
o->instance_name, FLAGS_instance_name, XLLM_META_STRING_FIELD_MAX_LEN);
CopyToFixed(o->kv_cache_transfer_mode,
FLAGS_kv_cache_transfer_mode,
XLLM_META_STRING_FIELD_MAX_LEN);
CopyToFixed(
o->log_dir, FLAGS_xllm_init_log_dir, XLLM_META_STRING_FIELD_MAX_LEN);
CopyToFixed(
o->draft_model, FLAGS_draft_model, XLLM_META_STRING_FIELD_MAX_LEN);
CopyToFixed(
o->draft_devices, FLAGS_draft_devices, XLLM_META_STRING_FIELD_MAX_LEN);
}
void PbToXllmDims(const c_api_test::XLLM_Dims& pb, XLLM_Dims* out) {
std::memset(out->dim, 0, sizeof(out->dim));
out->rank = pb.rank();
const int n = std::min(8, pb.dim_size());
for (int i = 0; i < n; ++i) {
out->dim[i] = pb.dim(i);
}
}
void XllmDimsToPb(const XLLM_Dims& in, c_api_test::XLLM_Dims* pb) {
pb->set_rank(in.rank);
pb->clear_dim();
const int n = std::min(8, in.rank);
for (int i = 0; i < n; ++i) {
pb->add_dim(in.dim[i]);
}
}
void PbToXllmTensor(const c_api_test::XLLM_Tensor& pb,
XLLM_Tensor* out,
MmDataOwned* owned) {
out->dtype = static_cast<XLLM_DataType>(pb.dtype());
PbToXllmDims(pb.dims(), &out->dims);
owned->tensor_byte_buffers.emplace_back(pb.data().begin(), pb.data().end());
std::vector<uint8_t>& buf = owned->tensor_byte_buffers.back();
out->data = buf.empty() ? nullptr : static_cast<const void*>(buf.data());
}
void XllmTensorToPb(const XLLM_Tensor& in, c_api_test::XLLM_Tensor* pb) {
pb->set_dtype(static_cast<c_api_test::XLLM_DataType>(in.dtype));
XllmDimsToPb(in.dims, pb->mutable_dims());
const size_t n = TensorNumElements(in.dims);
const size_t es = DTypeSize(in.dtype);
if (in.data != nullptr && n > 0 && es > 0) {
pb->set_data(static_cast<const char*>(in.data), n * es);
} else {
pb->clear_data();
}
}
void PbToXllmTensors(const c_api_test::XLLM_Tensors& pb,
XLLM_Tensors* out,
MmDataOwned* owned) {
owned->tensor_lists.emplace_back();
std::vector<XLLM_Tensor>& row = owned->tensor_lists.back();
row.reserve(static_cast<size_t>(pb.entries_size()));
for (int i = 0; i < pb.entries_size(); ++i) {
XLLM_Tensor t{};
PbToXllmTensor(pb.entries(i), &t, owned);
row.push_back(t);
}
out->entries = row.data();
out->entries_size = row.size();
}
void XllmTensorsToPb(const XLLM_Tensors& in, c_api_test::XLLM_Tensors* pb) {
pb->clear_entries();
for (size_t i = 0; i < in.entries_size; ++i) {
XllmTensorToPb(in.entries[i], pb->add_entries());
}
}
void PbToXllmMmValue(const c_api_test::XLLM_MM_Value& pb,
XLLM_MM_Value* out,
MmDataOwned* owned) {
std::memset(out, 0, sizeof(*out));
out->is_single_tensor = pb.is_single_tensor();
switch (pb.data_case()) {
case c_api_test::XLLM_MM_Value::kTensor:
out->is_single_tensor = true;
PbToXllmTensor(pb.tensor(), &out->data.tensor, owned);
break;
case c_api_test::XLLM_MM_Value::kTensors:
out->is_single_tensor = false;
PbToXllmTensors(pb.tensors(), &out->data.tensors, owned);
break;
default:
break;
}
}
void XllmMmValueToPb(const XLLM_MM_Value& in, c_api_test::XLLM_MM_Value* pb) {
pb->set_is_single_tensor(in.is_single_tensor);
if (in.is_single_tensor) {
XllmTensorToPb(in.data.tensor, pb->mutable_tensor());
} else {
XllmTensorsToPb(in.data.tensors, pb->mutable_tensors());
}
}
void PbToXllmMmDict(const c_api_test::XLLM_MM_Dict& pb,
XLLM_MM_Dict* out,
MmDataOwned* owned) {
owned->mm_dict_entries.clear();
owned->mm_dict_entries.reserve(static_cast<size_t>(pb.entries_size()));
for (int i = 0; i < pb.entries_size(); ++i) {
owned->mm_dict_entries.emplace_back();
XLLM_MM_DictEntry& e = owned->mm_dict_entries.back();
std::memset(e.key, 0, sizeof(e.key));
const std::string& k = pb.entries(i).key();
std::strncpy(e.key, k.c_str(), XLLM_META_STRING_FIELD_MAX_LEN - 1);
PbToXllmMmValue(pb.entries(i).value(), &e.value, owned);
}
out->entries = owned->mm_dict_entries.data();
out->entries_size = owned->mm_dict_entries.size();
}
void XllmMmDictToPb(const XLLM_MM_Dict& in, c_api_test::XLLM_MM_Dict* pb) {
pb->clear_entries();
for (size_t i = 0; i < in.entries_size; ++i) {
c_api_test::XLLM_MM_DictEntry* e = pb->add_entries();
e->set_key(in.entries[i].key);
XllmMmValueToPb(in.entries[i].value, e->mutable_value());
}
}
void PbToXllmMmItems(const c_api_test::XLLM_MM_Items& pb,
XLLM_MM_Items* out,
MmDataOwned* owned) {
owned->mm_items.clear();
owned->mm_items.reserve(static_cast<size_t>(pb.entries_size()));
for (int i = 0; i < pb.entries_size(); ++i) {
owned->mm_items.emplace_back();
PbToXllmMmItem(pb.entries(i), &owned->mm_items.back(), owned);
}
out->entries = owned->mm_items.data();
out->entries_size = owned->mm_items.size();
}
void XllmMmItemsToPb(const XLLM_MM_Items& in, c_api_test::XLLM_MM_Items* pb) {
pb->clear_entries();
for (size_t i = 0; i < in.entries_size; ++i) {
XllmMmItemToPb(in.entries[i], pb->add_entries());
}
}
void PbToXllmMmState(const c_api_test::XLLM_MM_State& pb, XLLM_MM_State* out) {
out->token_pos.offset = pb.token_pos().offset();
out->token_pos.length = pb.token_pos().length();
}
void XllmMmStateToPb(const XLLM_MM_State& in, c_api_test::XLLM_MM_State* pb) {
pb->mutable_token_pos()->set_offset(in.token_pos.offset);
pb->mutable_token_pos()->set_length(in.token_pos.length);
}
void PbToXllmMmItem(const c_api_test::XLLM_MM_Item& pb,
XLLM_MM_Item* out,
MmDataOwned* owned) {
std::memset(out, 0, sizeof(*out));
out->type = static_cast<XLLM_MM_Type>(pb.type());
PbToXllmMmValue(pb.data(), &out->data, owned);
PbToXllmMmState(pb.state(), &out->state);
}
void XllmMmItemToPb(const XLLM_MM_Item& in, c_api_test::XLLM_MM_Item* pb) {
pb->set_type(static_cast<uint32_t>(in.type));
XllmMmValueToPb(in.data, pb->mutable_data());
XllmMmStateToPb(in.state, pb->mutable_state());
}
bool PbToXllmMmData(const c_api_test::XLLM_MM_Data& pb,
XLLM_MM_Data* out,
MmDataOwned* owned) {
std::memset(out, 0, sizeof(*out));
out->type_mask = pb.type_mask();
out->is_dict = pb.is_dict();
switch (pb.storage_case()) {
case c_api_test::XLLM_MM_Data::kDict:
out->is_dict = true;
PbToXllmMmDict(pb.dict(), &out->data.dict, owned);
return true;
case c_api_test::XLLM_MM_Data::kItems:
out->is_dict = false;
PbToXllmMmItems(pb.items(), &out->data.items, owned);
return true;
default:
return false;
}
}
void XllmMmDataToPb(const XLLM_MM_Data& in, c_api_test::XLLM_MM_Data* pb) {
pb->set_type_mask(in.type_mask);
pb->set_is_dict(in.is_dict);
if (in.is_dict) {
XllmMmDictToPb(in.data.dict, pb->mutable_dict());
} else {
XllmMmItemsToPb(in.data.items, pb->mutable_items());
}
}
void PbToXllmRequestParams(const c_api_test::XLLM_RequestParams& pb,
XLLM_RequestParams* out) {
out->echo = pb.echo();
out->offline = pb.offline();
out->logprobs = pb.logprobs();
out->ignore_eos = pb.ignore_eos();
out->n = pb.n();
out->max_tokens = pb.max_tokens();
out->best_of = pb.best_of();
out->ttlt_slo_ms = pb.ttlt_slo_ms();
out->ttft_slo_ms = pb.ttft_slo_ms();
out->tpot_slo_ms = pb.tpot_slo_ms();
out->beam_width = pb.beam_width();
out->top_logprobs = pb.top_logprobs();
out->top_k = pb.top_k();
out->top_p = pb.top_p();
out->frequency_penalty = pb.frequency_penalty();
out->presence_penalty = pb.presence_penalty();
out->repetition_penalty = pb.repetition_penalty();
out->temperature = pb.temperature();
std::strncpy(out->request_id,
pb.request_id().c_str(),
XLLM_META_STRING_FIELD_MAX_LEN - 1);
out->request_id[XLLM_META_STRING_FIELD_MAX_LEN - 1] = '\0';
}
void XllmRequestParamsToPb(const XLLM_RequestParams& in,
c_api_test::XLLM_RequestParams* pb) {
pb->set_echo(in.echo);
pb->set_offline(in.offline);
pb->set_logprobs(in.logprobs);
pb->set_ignore_eos(in.ignore_eos);
pb->set_n(in.n);
pb->set_max_tokens(in.max_tokens);
pb->set_best_of(in.best_of);
pb->set_ttlt_slo_ms(in.ttlt_slo_ms);
pb->set_ttft_slo_ms(in.ttft_slo_ms);
pb->set_tpot_slo_ms(in.tpot_slo_ms);
pb->set_beam_width(in.beam_width);
pb->set_top_logprobs(in.top_logprobs);
pb->set_top_k(in.top_k);
pb->set_top_p(in.top_p);
pb->set_frequency_penalty(in.frequency_penalty);
pb->set_presence_penalty(in.presence_penalty);
pb->set_repetition_penalty(in.repetition_penalty);
pb->set_temperature(in.temperature);
pb->set_request_id(in.request_id);
}
void PbToXllmChatMessage(const c_api_test::XLLM_ChatMessage& pb,
XLLM_ChatMessage* out) {
std::memset(out->role, 0, sizeof(out->role));
std::strncpy(
out->role, pb.role().c_str(), XLLM_META_STRING_FIELD_MAX_LEN - 1);
out->role[XLLM_META_STRING_FIELD_MAX_LEN - 1] = '\0';
if (!pb.content().empty()) {
out->content = new char[pb.content().size() + 1];
std::memcpy(out->content, pb.content().data(), pb.content().size());
out->content[pb.content().size()] = '\0';
} else {
out->content = nullptr;
}
}
void FreeXllmChatMessageContent(XLLM_ChatMessage* out) {
delete[] out->content;
out->content = nullptr;
}
void XllmChatMessageToPb(const XLLM_ChatMessage* in,
c_api_test::XLLM_ChatMessage* pb) {
if (!in) {
return;
}
pb->set_role(in->role);
if (in->content != nullptr) {
pb->set_content(in->content);
} else {
pb->clear_content();
}
}
void PbToXllmUsage(const c_api_test::XLLM_Usage& pb, XLLM_Usage* out) {
out->prompt_tokens = pb.prompt_tokens();
out->completion_tokens = pb.completion_tokens();
out->total_tokens = pb.total_tokens();
}
void XllmUsageToPb(const XLLM_Usage& in, c_api_test::XLLM_Usage* pb) {
pb->set_prompt_tokens(in.prompt_tokens);
pb->set_completion_tokens(in.completion_tokens);
pb->set_total_tokens(in.total_tokens);
}
void PbToXllmLogProbs(const c_api_test::XLLM_LogProbs& pb,
XLLM_LogProbs* out,
std::vector<XLLM_LogProb>* storage) {
storage->clear();
storage->reserve(static_cast<size_t>(pb.entries_size()));
for (int i = 0; i < pb.entries_size(); ++i) {
XLLM_LogProb e{};
e.token_id = pb.entries(i).token_id();
e.logprob = pb.entries(i).logprob();
storage->push_back(e);
}
out->entries = storage->empty() ? nullptr : storage->data();
out->entries_size = storage->size();
}
void XllmLogProbsToPb(const XLLM_LogProbs& in, c_api_test::XLLM_LogProbs* pb) {
pb->clear_entries();
if (in.entries == nullptr || in.entries_size == 0) {
return;
}
for (size_t i = 0; i < in.entries_size; ++i) {
c_api_test::XLLM_LogProb* e = pb->add_entries();
e->set_token_id(in.entries[i].token_id);
e->set_logprob(in.entries[i].logprob);
}
}
void PbToXllmChoice(const c_api_test::XLLM_Choice& pb,
XLLM_Choice* out,
ResponseOwned* ro) {
std::memset(out, 0, sizeof(*out));
out->index = pb.index();
if (!pb.text().empty()) {
ro->choice_text_bufs.push_back(CopyCStr(pb.text()));
out->text = ro->choice_text_bufs.back().get();
}
if (pb.has_chat_message()) {
ro->chat_messages.emplace_back();
XLLM_ChatMessage& cm = ro->chat_messages.back();
std::memset(cm.role, 0, sizeof(cm.role));
std::strncpy(cm.role,
pb.chat_message().role().c_str(),
XLLM_META_STRING_FIELD_MAX_LEN - 1);
cm.role[XLLM_META_STRING_FIELD_MAX_LEN - 1] = '\0';
if (!pb.chat_message().content().empty()) {
ro->chat_message_contents.push_back(
CopyCStr(pb.chat_message().content()));
cm.content = ro->chat_message_contents.back().get();
} else {
cm.content = nullptr;
}
out->message = &ro->chat_messages.back();
}
ro->token_ids_vecs.emplace_back();
ro->token_ids_vecs.back().reserve(static_cast<size_t>(pb.token_ids_size()));
for (int i = 0; i < pb.token_ids_size(); ++i) {
ro->token_ids_vecs.back().push_back(pb.token_ids(i));
}
out->token_ids = ro->token_ids_vecs.back().data();
out->token_size = ro->token_ids_vecs.back().size();
ro->logprob_vecs.emplace_back();
std::vector<XLLM_LogProb>& le = ro->logprob_vecs.back();
le.reserve(static_cast<size_t>(pb.logprobs().entries_size()));
for (int i = 0; i < pb.logprobs().entries_size(); ++i) {
XLLM_LogProb e{};
e.token_id = pb.logprobs().entries(i).token_id();
e.logprob = pb.logprobs().entries(i).logprob();
le.push_back(e);
}
out->logprobs.entries = le.data();
out->logprobs.entries_size = le.size();
std::strncpy(out->finish_reason,
pb.finish_reason().c_str(),
XLLM_META_STRING_FIELD_MAX_LEN - 1);
out->finish_reason[XLLM_META_STRING_FIELD_MAX_LEN - 1] = '\0';
}
void XllmChoiceToPb(const XLLM_Choice& in, c_api_test::XLLM_Choice* pb) {
pb->set_index(in.index);
if (in.text != nullptr) {
pb->set_text(in.text);
} else {
pb->clear_text();
}
if (in.message != nullptr) {
XllmChatMessageToPb(in.message, pb->mutable_chat_message());
} else {
pb->clear_chat_message();
}
pb->clear_token_ids();
if (in.token_ids != nullptr) {
for (size_t i = 0; i < in.token_size; ++i) {
pb->add_token_ids(in.token_ids[i]);
}
}
XllmLogProbsToPb(in.logprobs, pb->mutable_logprobs());
pb->set_finish_reason(in.finish_reason);
}
void PbToXllmChoices(const c_api_test::XLLM_Choices& pb,
XLLM_Choices* out,
ResponseOwned* ro) {
ro->choices.clear();
ro->choices.reserve(static_cast<size_t>(pb.entries_size()));
for (int i = 0; i < pb.entries_size(); ++i) {
ro->choices.emplace_back();
PbToXllmChoice(pb.entries(i), &ro->choices.back(), ro);
}
out->entries = ro->choices.data();
out->entries_size = ro->choices.size();
}
void XllmChoicesToPb(const XLLM_Choices& in, c_api_test::XLLM_Choices* pb) {
pb->clear_entries();
if (in.entries == nullptr) {
return;
}
for (size_t i = 0; i < in.entries_size; ++i) {
XllmChoiceToPb(in.entries[i], pb->add_entries());
}
}
void PbToXllmResponse(const c_api_test::XLLM_Response& pb,
XLLM_Response* out,
ResponseOwned* owned) {
std::memset(out, 0, sizeof(*out));
out->status_code = static_cast<XLLM_StatusCode>(pb.status_code());
std::strncpy(
out->error_info, pb.error_info().c_str(), XLLM_ERROR_INFO_MAX_LEN - 1);
out->error_info[XLLM_ERROR_INFO_MAX_LEN - 1] = '\0';
std::strncpy(out->id, pb.id().c_str(), XLLM_META_STRING_FIELD_MAX_LEN - 1);
out->id[XLLM_META_STRING_FIELD_MAX_LEN - 1] = '\0';
std::strncpy(
out->object, pb.object().c_str(), XLLM_META_STRING_FIELD_MAX_LEN - 1);
out->object[XLLM_META_STRING_FIELD_MAX_LEN - 1] = '\0';
out->created = pb.created();
std::strncpy(
out->model, pb.model().c_str(), XLLM_META_STRING_FIELD_MAX_LEN - 1);
out->model[XLLM_META_STRING_FIELD_MAX_LEN - 1] = '\0';
PbToXllmUsage(pb.usage(), &out->usage);
PbToXllmChoices(pb.choices(), &out->choices, owned);
}
void XllmResponseToPb(const XLLM_Response* in, c_api_test::XLLM_Response* pb) {
if (!in) {
pb->Clear();
return;
}
pb->set_status_code(
static_cast<c_api_test::XLLM_StatusCode>(in->status_code));
pb->set_error_info(in->error_info);
pb->set_id(in->id);
pb->set_object(in->object);
pb->set_created(in->created);
pb->set_model(in->model);
XllmUsageToPb(in->usage, pb->mutable_usage());
XllmChoicesToPb(in->choices, pb->mutable_choices());
}
} // namespace xllm_capi_test

146
upstream_ref/xllm/xllm/c_api/test/utils.h Normal file
View File

@@ -0,0 +1,146 @@
/* Copyright 2026 The xLLM Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
https://github.com/jd-opensource/xllm/blob/main/LICENSE
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef XLLM_C_API_TEST_UTILS_H_
#define XLLM_C_API_TEST_UTILS_H_
#include <gflags/gflags.h>
#include <cstddef>
#include <memory>
#include <vector>
#include "types.h"
#include "xllm_test.pb.h"
DECLARE_string(model_path);
DECLARE_string(devices);
DECLARE_int32(port);
DECLARE_string(listen_addr);
DECLARE_int32(idle_timeout_s);
DECLARE_string(backend);
namespace xllm_capi_test {
// Applies gflags (after ParseCommandLineFlags) into XLLM_InitOptions.
void ApplyGflagsToXllmInitOptions(XLLM_InitOptions* opt);
// Owns buffers referenced by XLLM_MM_Data after PbToXllmMmData.
struct MmDataOwned {
std::vector<std::vector<uint8_t>> tensor_byte_buffers;
std::vector<std::vector<XLLM_Tensor>> tensor_lists;
std::vector<XLLM_MM_Item> mm_items;
std::vector<XLLM_MM_DictEntry> mm_dict_entries;
};
// Owns heap data for XLLM_Response filled by PbToXllmResponse (text, message,
// token_ids, logprobs arrays).
struct ResponseOwned {
std::vector<std::unique_ptr<char[]>> choice_text_bufs;
std::vector<XLLM_ChatMessage> chat_messages;
std::vector<std::unique_ptr<char[]>> chat_message_contents;
std::vector<std::vector<int32_t>> token_ids_vecs;
std::vector<std::vector<XLLM_LogProb>> logprob_vecs;
std::vector<XLLM_Choice> choices;
};
void PbToXllmRequestParams(const c_api_test::XLLM_RequestParams& pb,
XLLM_RequestParams* out);
void XllmRequestParamsToPb(const XLLM_RequestParams& in,
c_api_test::XLLM_RequestParams* pb);
void PbToXllmChatMessage(const c_api_test::XLLM_ChatMessage& pb,
XLLM_ChatMessage* out);
void FreeXllmChatMessageContent(XLLM_ChatMessage* out);
void XllmChatMessageToPb(const XLLM_ChatMessage* in,
c_api_test::XLLM_ChatMessage* pb);
bool PbToXllmMmData(const c_api_test::XLLM_MM_Data& pb,
XLLM_MM_Data* out,
MmDataOwned* owned);
void XllmMmDataToPb(const XLLM_MM_Data& in, c_api_test::XLLM_MM_Data* pb);
void XllmResponseToPb(const XLLM_Response* in, c_api_test::XLLM_Response* pb);
void PbToXllmResponse(const c_api_test::XLLM_Response& pb,
XLLM_Response* out,
ResponseOwned* owned);
// --- Lower-level (types.h <-> pb) ---
void PbToXllmDims(const c_api_test::XLLM_Dims& pb, XLLM_Dims* out);
void XllmDimsToPb(const XLLM_Dims& in, c_api_test::XLLM_Dims* pb);
void PbToXllmTensor(const c_api_test::XLLM_Tensor& pb,
XLLM_Tensor* out,
MmDataOwned* owned);
void XllmTensorToPb(const XLLM_Tensor& in, c_api_test::XLLM_Tensor* pb);
void PbToXllmTensors(const c_api_test::XLLM_Tensors& pb,
XLLM_Tensors* out,
MmDataOwned* owned);
void XllmTensorsToPb(const XLLM_Tensors& in, c_api_test::XLLM_Tensors* pb);
void PbToXllmMmValue(const c_api_test::XLLM_MM_Value& pb,
XLLM_MM_Value* out,
MmDataOwned* owned);
void XllmMmValueToPb(const XLLM_MM_Value& in, c_api_test::XLLM_MM_Value* pb);
void PbToXllmMmDict(const c_api_test::XLLM_MM_Dict& pb,
XLLM_MM_Dict* out,
MmDataOwned* owned);
void XllmMmDictToPb(const XLLM_MM_Dict& in, c_api_test::XLLM_MM_Dict* pb);
void PbToXllmMmItems(const c_api_test::XLLM_MM_Items& pb,
XLLM_MM_Items* out,
MmDataOwned* owned);
void XllmMmItemsToPb(const XLLM_MM_Items& in, c_api_test::XLLM_MM_Items* pb);
void PbToXllmMmState(const c_api_test::XLLM_MM_State& pb, XLLM_MM_State* out);
void XllmMmStateToPb(const XLLM_MM_State& in, c_api_test::XLLM_MM_State* pb);
void PbToXllmMmItem(const c_api_test::XLLM_MM_Item& pb,
XLLM_MM_Item* out,
MmDataOwned* owned);
void XllmMmItemToPb(const XLLM_MM_Item& in, c_api_test::XLLM_MM_Item* pb);
void PbToXllmUsage(const c_api_test::XLLM_Usage& pb, XLLM_Usage* out);
void XllmUsageToPb(const XLLM_Usage& in, c_api_test::XLLM_Usage* pb);
void PbToXllmLogProbs(const c_api_test::XLLM_LogProbs& pb,
XLLM_LogProbs* out,
std::vector<XLLM_LogProb>* storage);
void XllmLogProbsToPb(const XLLM_LogProbs& in, c_api_test::XLLM_LogProbs* pb);
void PbToXllmChoice(const c_api_test::XLLM_Choice& pb,
XLLM_Choice* out,
ResponseOwned* owned);
void XllmChoiceToPb(const XLLM_Choice& in, c_api_test::XLLM_Choice* pb);
void PbToXllmChoices(const c_api_test::XLLM_Choices& pb,
XLLM_Choices* out,
ResponseOwned* owned);
void XllmChoicesToPb(const XLLM_Choices& in, c_api_test::XLLM_Choices* pb);
} // namespace xllm_capi_test
#endif // XLLM_C_API_TEST_UTILS_H_

320
upstream_ref/xllm/xllm/c_api/test/xllm_test.cpp Normal file
View File

@@ -0,0 +1,320 @@
/* Copyright 2026 The xLLM Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
https://github.com/jd-opensource/xllm/blob/main/LICENSE
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#include <brpc/controller.h>
#include <brpc/server.h>
#include <butil/endpoint.h>
#include <butil/logging.h>
#include <gflags/gflags.h>
#include <cctype>
#include <cstddef>
#include <cstdint>
#include <cstring>
#include <memory>
#include <mutex>
#include <string>
#include <vector>
#include "llm.h"
#include "rec.h"
#include "utils.h"
#include "xllm_test.pb.h"
namespace xllm_capi_test {
namespace {
std::unique_ptr<char[]> CopyContent(const std::string& s) {
if (s.empty()) {
return nullptr;
}
auto p = std::make_unique<char[]>(s.size() + 1);
std::memcpy(p.get(), s.data(), s.size());
p[s.size()] = '\0';
return p;
}
void SetErrorResponse(c_api_test::XLLM_Response* res,
c_api_test::XLLM_StatusCode code,
const std::string& msg) {
res->Clear();
res->set_status_code(code);
res->set_error_info(msg);
}
} // namespace
class XllmRecCapiServiceImpl : public c_api_test::XllmRecCapiService {
public:
XllmRecCapiServiceImpl(XLLM_REC_Handler* rec_handler,
XLLM_LLM_Handler* llm_handler)
: rec_handler_(rec_handler), llm_handler_(llm_handler) {}
void Inference(google::protobuf::RpcController* cntl_base,
const c_api_test::XLLM_Request* request,
c_api_test::XLLM_Response* response,
google::protobuf::Closure* done) override {
brpc::ClosureGuard done_guard(done);
(void)cntl_base;
std::lock_guard<std::mutex> lock(mu_);
const std::string& fn = request->call_function();
const bool is_llm_fn =
(fn == "xllm_llm_completions" || fn == "xllm_llm_chat_completions");
const bool is_rec_fn = (fn == "xllm_rec_text_completions" ||
fn == "xllm_rec_token_completions" ||
fn == "xllm_rec_multimodal_completions" ||
fn == "xllm_rec_chat_completions");
if (is_llm_fn && !llm_handler_) {
SetErrorResponse(response,
c_api_test::XLLM_STATUS_INTERNAL_ERROR,
"LLM handler is null");
return;
}
if (is_rec_fn && !rec_handler_) {
SetErrorResponse(response,
c_api_test::XLLM_STATUS_INTERNAL_ERROR,
"REC handler is null");
return;
}
XLLM_RequestParams params{};
if (is_llm_fn) {
xllm_llm_request_params_default(&params);
} else {
xllm_rec_request_params_default(&params);
}
if (request->params().ByteSizeLong() > 0) {
PbToXllmRequestParams(request->params(), &params);
}
const char* model_id =
request->model_id().empty() ? "" : request->model_id().c_str();
const uint32_t timeout_ms = request->timeout_ms();
XLLM_Response* raw = nullptr;
if (fn == "xllm_llm_completions") {
raw = xllm_llm_completions(llm_handler_,
model_id,
request->prompt().c_str(),
timeout_ms,
&params);
} else if (fn == "xllm_llm_chat_completions") {
std::vector<XLLM_ChatMessage> cms;
std::vector<std::unique_ptr<char[]>> contents;
cms.reserve(request->messages_size());
for (int i = 0; i < request->messages_size(); ++i) {
const auto& m = request->messages(i);
XLLM_ChatMessage cm{};
std::memset(cm.role, 0, sizeof(cm.role));
std::strncpy(
cm.role, m.role().c_str(), XLLM_META_STRING_FIELD_MAX_LEN - 1);
cm.role[XLLM_META_STRING_FIELD_MAX_LEN - 1] = '\0';
contents.push_back(CopyContent(m.content()));
cm.content = contents.back() ? contents.back().get() : nullptr;
cms.push_back(cm);
}
raw = xllm_llm_chat_completions(llm_handler_,
model_id,
cms.empty() ? nullptr : cms.data(),
cms.size(),
timeout_ms,
&params);
} else if (fn == "xllm_rec_text_completions") {
raw = xllm_rec_text_completions(rec_handler_,
model_id,
request->prompt().c_str(),
timeout_ms,
&params);
} else if (fn == "xllm_rec_token_completions") {
std::vector<int32_t> token_ids;
token_ids.reserve(request->token_ids_size());
for (int i = 0; i < request->token_ids_size(); ++i) {
token_ids.push_back(request->token_ids(i));
}
raw = xllm_rec_token_completions(
rec_handler_,
model_id,
token_ids.empty() ? nullptr : token_ids.data(),
token_ids.size(),
timeout_ms,
&params);
} else if (fn == "xllm_rec_multimodal_completions") {
std::vector<int32_t> token_ids;
token_ids.reserve(request->token_ids_size());
for (int i = 0; i < request->token_ids_size(); ++i) {
token_ids.push_back(request->token_ids(i));
}
XLLM_MM_Data mm{};
MmDataOwned mm_owned;
if (!PbToXllmMmData(request->mm_data(), &mm, &mm_owned)) {
SetErrorResponse(response,
c_api_test::XLLM_STATUS_INVALID_REQUEST,
"invalid or empty mm_data");
return;
}
raw = xllm_rec_multimodal_completions(
rec_handler_,
model_id,
token_ids.empty() ? nullptr : token_ids.data(),
token_ids.size(),
&mm,
timeout_ms,
&params);
} else if (fn == "xllm_rec_chat_completions") {
std::vector<XLLM_ChatMessage> cms;
std::vector<std::unique_ptr<char[]>> contents;
cms.reserve(request->messages_size());
for (int i = 0; i < request->messages_size(); ++i) {
const auto& m = request->messages(i);
XLLM_ChatMessage cm{};
std::memset(cm.role, 0, sizeof(cm.role));
std::strncpy(
cm.role, m.role().c_str(), XLLM_META_STRING_FIELD_MAX_LEN - 1);
cm.role[XLLM_META_STRING_FIELD_MAX_LEN - 1] = '\0';
contents.push_back(CopyContent(m.content()));
cm.content = contents.back() ? contents.back().get() : nullptr;
cms.push_back(cm);
}
raw = xllm_rec_chat_completions(rec_handler_,
model_id,
cms.empty() ? nullptr : cms.data(),
cms.size(),
timeout_ms,
&params);
} else {
SetErrorResponse(response,
c_api_test::XLLM_STATUS_INVALID_REQUEST,
"unsupported call_function: " + fn);
return;
}
if (raw == nullptr) {
SetErrorResponse(response,
c_api_test::XLLM_STATUS_INTERNAL_ERROR,
"C API returned null response");
return;
}
XllmResponseToPb(raw, response);
if (is_llm_fn) {
xllm_llm_free_response(raw);
} else {
xllm_rec_free_response(raw);
}
}
private:
XLLM_REC_Handler* rec_handler_;
XLLM_LLM_Handler* llm_handler_;
std::mutex mu_;
};
} // namespace xllm_capi_test
int main(int argc, char* argv[]) {
GFLAGS_NAMESPACE::ParseCommandLineFlags(&argc, &argv, true);
if (FLAGS_model_path.empty()) {
LOG(ERROR) << "Missing --model_path (set in gflags file or command line)";
return -1;
}
std::string backend = FLAGS_backend;
for (char& c : backend) {
c = static_cast<char>(std::tolower(static_cast<unsigned char>(c)));
}
if (backend != "llm" && backend != "rec") {
LOG(ERROR) << "Invalid --backend=\"" << FLAGS_backend
<< "\" (expected llm or rec)";
return -2;
}
XLLM_REC_Handler* rec_ptr = nullptr;
XLLM_LLM_Handler* llm_ptr = nullptr;
std::unique_ptr<XLLM_REC_Handler, void (*)(XLLM_REC_Handler*)> rec_holder(
nullptr, xllm_rec_destroy);
std::unique_ptr<XLLM_LLM_Handler, void (*)(XLLM_LLM_Handler*)> llm_holder(
nullptr, xllm_llm_destroy);
if (backend == "rec") {
rec_holder.reset(xllm_rec_create());
if (!rec_holder) {
LOG(ERROR) << "xllm_rec_create failed";
return -3;
}
XLLM_InitOptions init{};
xllm_rec_init_options_default(&init);
xllm_capi_test::ApplyGflagsToXllmInitOptions(&init);
if (!xllm_rec_initialize(rec_holder.get(),
FLAGS_model_path.c_str(),
FLAGS_devices.c_str(),
&init)) {
LOG(ERROR) << "xllm_rec_initialize failed model_path=" << FLAGS_model_path
<< " devices=" << FLAGS_devices;
return -4;
}
rec_ptr = rec_holder.get();
} else {
llm_holder.reset(xllm_llm_create());
if (!llm_holder) {
LOG(ERROR) << "xllm_llm_create failed";
return -5;
}
XLLM_InitOptions init{};
xllm_llm_init_options_default(&init);
xllm_capi_test::ApplyGflagsToXllmInitOptions(&init);
if (!xllm_llm_initialize(llm_holder.get(),
FLAGS_model_path.c_str(),
FLAGS_devices.c_str(),
&init)) {
LOG(ERROR) << "xllm_llm_initialize failed model_path=" << FLAGS_model_path
<< " devices=" << FLAGS_devices;
return -6;
}
llm_ptr = llm_holder.get();
}
xllm_capi_test::XllmRecCapiServiceImpl svc(rec_ptr, llm_ptr);
brpc::Server server;
if (server.AddService(&svc, brpc::SERVER_DOESNT_OWN_SERVICE) != 0) {
LOG(ERROR) << "Fail to add XllmRecCapiService";
return -7;
}
butil::EndPoint point;
if (!FLAGS_listen_addr.empty()) {
if (butil::str2endpoint(FLAGS_listen_addr.c_str(), &point) < 0) {
LOG(ERROR) << "Invalid --listen_addr=" << FLAGS_listen_addr;
return -8;
}
} else {
point = butil::EndPoint(butil::IP_ANY, FLAGS_port);
}
brpc::ServerOptions options;
options.idle_timeout_sec = FLAGS_idle_timeout_s;
if (server.Start(point, &options) != 0) {
LOG(ERROR) << "Fail to start brpc server";
return -9;
}
LOG(INFO) << "xllm_test C API brpc server backend=" << backend
<< " listening on " << butil::endpoint2str(point).c_str();
server.RunUntilAskedToQuit();
return 0;
}

24
upstream_ref/xllm/xllm/c_api/test/xllm_test.flags Normal file
View File

@@ -0,0 +1,24 @@
# Example gflags for xllm_test brpc C API server.
# Start: xllm_test --flagfile=xllm/c_api/test/xllm_test.flags
#
# --- Backend: only one of llm (c_api/llm.h) or rec (c_api/rec.h) is loaded ---
--backend=rec
# --backend=llm
#
# --- Required for xllm_*_initialize ---
--model_path=/export/home/models/Qwen3-8B
--devices=npu:4
#
# --- brpc listen (optional) ---
--port=8000
# --listen_addr=0.0.0.0:8000
# --idle_timeout_s=-1
#
# --- XLLM_InitOptions (override as needed; defaults match REC) ---
# --enable_chunked_prefill=false
# --transfer_listen_port=26000
# --max_memory_utilization=0.55
# --init_task=generate
# --communication_backend=lccl
# --master_node_addr=127.0.0.1:18899
# --xllm_init_log_dir= # maps to XLLM_InitOptions.log_dir (not glog's --log_dir)

224
upstream_ref/xllm/xllm/c_api/test/xllm_test.proto Normal file
View File

@@ -0,0 +1,224 @@
syntax = "proto3";
package c_api_test;
option cc_generic_services = true;
// =============================================================================
// Mirrors xllm/c_api/types.h (all public structs/enums) except:
// - XLLM_InitOptions / XLLM_InitLLMOptions are intentionally omitted.
// =============================================================================
// --- XLLM_DataType ---
enum XLLM_DataType {
XLLM_DTYPE_UNDEFINED = 0;
XLLM_DTYPE_FLOAT16 = 1;
XLLM_DTYPE_FLOAT32 = 2;
XLLM_DTYPE_FLOAT64 = 3;
XLLM_DTYPE_BFLOAT16 = 4;
XLLM_DTYPE_INT8 = 5;
XLLM_DTYPE_INT16 = 6;
XLLM_DTYPE_INT32 = 7;
XLLM_DTYPE_INT64 = 8;
XLLM_DTYPE_UINT8 = 9;
XLLM_DTYPE_UINT16 = 10;
XLLM_DTYPE_UINT32 = 11;
XLLM_DTYPE_UINT64 = 12;
XLLM_DTYPE_BOOL = 13;
XLLM_DTYPE_STRING = 14;
}
// --- XLLM_StatusCode ---
enum XLLM_StatusCode {
XLLM_STATUS_SUCCESS = 0;
XLLM_STATUS_NOT_INITIALIZED = 1;
XLLM_STATUS_MODEL_NOT_FOUND = 2;
XLLM_STATUS_TIMEOUT = 3;
XLLM_STATUS_INVALID_REQUEST = 4;
XLLM_STATUS_INTERNAL_ERROR = 5;
}
// --- XLLM_MM_Type (same numeric values as C bitmask enum) ---
enum XLLM_MM_Type {
XLLM_MM_TYPE_NONE = 0;
XLLM_MM_TYPE_IMAGE = 1;
XLLM_MM_TYPE_AUDIO = 2;
XLLM_MM_TYPE_VIDEO = 4;
XLLM_MM_TYPE_TEXT = 8;
XLLM_MM_TYPE_EMBEDDING = 16;
}
// --- XLLM_Dims ---
message XLLM_Dims {
int32 rank = 1;
repeated int32 dim = 2;
}
// --- XLLM_Tensor ---
message XLLM_Tensor {
XLLM_DataType dtype = 1;
XLLM_Dims dims = 2;
bytes data = 3;
}
// --- XLLM_Tensors ---
message XLLM_Tensors {
repeated XLLM_Tensor entries = 1;
}
// --- XLLM_MM_Value ---
message XLLM_MM_Value {
bool is_single_tensor = 1;
oneof data {
XLLM_Tensor tensor = 2;
XLLM_Tensors tensors = 3;
}
}
// --- XLLM_MM_Meta (placeholder, matches empty struct in types.h) ---
message XLLM_MM_Meta {}
// --- XLLM_MM_TokenPos ---
message XLLM_MM_TokenPos {
uint32 offset = 1;
uint32 length = 2;
}
// --- XLLM_MM_State ---
message XLLM_MM_State {
XLLM_MM_TokenPos token_pos = 1;
}
// --- XLLM_MM_DictEntry ---
message XLLM_MM_DictEntry {
string key = 1;
XLLM_MM_Value value = 2;
}
// --- XLLM_MM_Dict ---
message XLLM_MM_Dict {
repeated XLLM_MM_DictEntry entries = 1;
}
// --- XLLM_MM_Item ---
message XLLM_MM_Item {
uint32 type = 1;
XLLM_MM_Value data = 2;
XLLM_MM_Meta meta = 3;
XLLM_MM_State state = 4;
}
// --- XLLM_MM_Items ---
message XLLM_MM_Items {
repeated XLLM_MM_Item entries = 1;
}
// --- XLLM_MM_Data ---
message XLLM_MM_Data {
uint32 type_mask = 1;
bool is_dict = 2;
oneof storage {
XLLM_MM_Dict dict = 3;
XLLM_MM_Items items = 4;
}
}
// --- XLLM_ChatMessage ---
message XLLM_ChatMessage {
string role = 1;
string content = 2;
}
// --- XLLM_RequestParams ---
message XLLM_RequestParams {
bool echo = 1;
bool offline = 2;
bool logprobs = 3;
bool ignore_eos = 4;
uint32 n = 5;
uint32 max_tokens = 6;
uint32 best_of = 7;
int32 ttlt_slo_ms = 8;
int32 ttft_slo_ms = 9;
int32 tpot_slo_ms = 10;
uint32 beam_width = 11;
int64 top_logprobs = 12;
int64 top_k = 13;
float top_p = 14;
float frequency_penalty = 15;
float presence_penalty = 16;
float repetition_penalty = 17;
float temperature = 18;
string request_id = 19;
}
// --- XLLM_Usage ---
message XLLM_Usage {
int32 prompt_tokens = 1;
int32 completion_tokens = 2;
int32 total_tokens = 3;
}
// --- XLLM_LogProb / XLLM_LogProbs ---
message XLLM_LogProb {
uint32 token_id = 1;
float logprob = 2;
}
message XLLM_LogProbs {
repeated XLLM_LogProb entries = 1;
}
// --- XLLM_Choice / XLLM_Choices ---
message XLLM_Choice {
uint32 index = 1;
string text = 2;
XLLM_ChatMessage chat_message = 3;
repeated int32 token_ids = 4;
XLLM_LogProbs logprobs = 5;
string finish_reason = 6;
}
message XLLM_Choices {
repeated XLLM_Choice entries = 1;
}
// --- XLLM_Response ---
message XLLM_Response {
XLLM_StatusCode status_code = 1;
string error_info = 2;
string id = 3;
string object = 4;
int64 created = 5;
string model = 6;
XLLM_Choices choices = 7;
XLLM_Usage usage = 8;
}
message XLLM_Request {
string call_function = 1;
uint32 timeout_ms = 2;
string prompt = 3;
repeated XLLM_ChatMessage messages = 4;
repeated int32 token_ids = 5;
XLLM_MM_Data mm_data = 6;
XLLM_RequestParams params = 7;
string model_id = 8;
}
// One dump file record: request + optional response (xllm_dump).
message XLLM_DumpRecord {
XLLM_Request request = 1;
XLLM_Response response = 2;
}
// brpc service: dispatch by XLLM_Request.call_function (must match xllm_test
// --backend: rec -> xllm_rec_*; llm -> xllm_llm_*).
service XllmRecCapiService {
rpc Inference(XLLM_Request) returns (XLLM_Response);
}

118
upstream_ref/xllm/xllm/c_api/tools/install.sh Executable file
View File

@@ -0,0 +1,118 @@
#!/bin/bash
set -euo pipefail
SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
BIN_DIR="${SCRIPT_DIR}/../../../bin"
TEMP_DIR="xllm"
INCLUDE_DIR="${TEMP_DIR}/include"
LIB_DIR="${TEMP_DIR}/lib"
VERSION_FILE="${SCRIPT_DIR}/../../../version.txt"
TAR_BASE_NAME="xllm"
LOCAL_INSTALL_DIR="/usr/local"
LOCAL_TARGET_DIR="${LOCAL_INSTALL_DIR}/xllm"
HEADERS=("${SCRIPT_DIR}/../llm.h" "${SCRIPT_DIR}/../rec.h" "${SCRIPT_DIR}/../default.h" "${SCRIPT_DIR}/../types.h")
SO_FILES=(
"${SCRIPT_DIR}/../../../build/xllm/core/server/libxllm.so"
)
error_exit() {
echo -e "\033[31merror: $1\033[0m" >&2
exit 1
}
cd_bin_dir() {
if [ ! -d "${BIN_DIR}" ]; then
mkdir -p "${BIN_DIR}" || error_exit "failed to create bin directory: ${BIN_DIR}"
fi
cd "${BIN_DIR}" || error_exit "failed to enter bin directory: ${BIN_DIR}"
}
read_version() {
if [ ! -f "${VERSION_FILE}" ]; then
error_exit "${VERSION_FILE} is not existed"
fi
VERSION=$(cat "${VERSION_FILE}" | tr -d '[:space:]')
if [ -z "${VERSION}" ]; then
error_exit "version content is empty"
fi
TAR_FILE="${TAR_BASE_NAME}_${VERSION}.tar.gz"
}
check_files() {
for header in "${HEADERS[@]}"; do
if [ ! -f "${header}" ]; then
error_exit "${header} is not existed"
fi
done
for so_file in "${SO_FILES[@]}"; do
if [ ! -f "${so_file}" ]; then
error_exit "${so_file} is not existed"
fi
done
}
create_dirs() {
mkdir -p "${INCLUDE_DIR}" || error_exit "create include directory failed"
mkdir -p "${LIB_DIR}" || error_exit "create lib directory failed"
}
copy_headers() {
for header in "${HEADERS[@]}"; do
cp -f "${header}" "${INCLUDE_DIR}/" || error_exit "copy ${header} failed"
done
}
copy_so() {
for so_file in "${SO_FILES[@]}"; do
cp -f "${so_file}" "${LIB_DIR}/" || error_exit "copy ${so_file} failed"
done
}
package_tar() {
tar -czf "${TAR_FILE}" "${TEMP_DIR}" || error_exit "tar failed"
}
cleanup_temp() {
rm -rf "${TEMP_DIR}" || error_exit "rm temp directory failed"
}
extract_to_local() {
if [ ! -f "${TAR_FILE}" ]; then
error_exit "${TAR_FILE} is not existed"
fi
if [ ! -d "${LOCAL_INSTALL_DIR}" ]; then
error_exit "local install directory is not existed"
fi
if [ -d "${LOCAL_TARGET_DIR}" ]; then
rm -rf "${LOCAL_TARGET_DIR}" || error_exit "rm old xllm directory failed"
fi
tar -xzf "${TAR_FILE}" -C "${LOCAL_INSTALL_DIR}" || error_exit "extract failed"
}
main() {
cd_bin_dir
read_version
check_files
create_dirs
copy_headers
copy_so
package_tar
cleanup_temp
extract_to_local
echo -e "install file: \033[33m${TAR_FILE}\033[0m"
echo -e "install path: \033[33m/usr/local/${TEMP_DIR}\033[0m"
}
main

627
upstream_ref/xllm/xllm/c_api/types.h Normal file
View File

@@ -0,0 +1,627 @@
/* Copyright 2025 The xLLM Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
https://github.com/jd-opensource/xllm/blob/main/LICENSE
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef XLLM_C_TYPES_H
#define XLLM_C_TYPES_H
#ifdef __cplusplus
extern "C" {
#endif
#include <stdbool.h>
#include <stddef.h>
#include <stdint.h>
// Maximum length for meta string fields (includes '\0' terminator)
#define XLLM_META_STRING_FIELD_MAX_LEN 128
// Export Macro Definition
#ifndef XLLM_CAPI_EXPORT
#define XLLM_CAPI_EXPORT __attribute__((visibility("default")))
#endif
// Core Struct & Enum Definitions
/**
* @brief Configuration options for initializing an LLM instance
* @note All string fields are fixed-length arrays. Default values are defined
* in macros. Empty string indicates disable/use default value.
*/
typedef struct XLLM_CAPI_EXPORT XLLM_InitOptions {
/** Whether to enable chunked prefill for inference */
bool enable_chunked_prefill;
/** Whether to enable prefill-only sequence parallel */
bool enable_prefill_sp;
/** Whether to enable prefix cache optimization */
bool enable_prefix_cache;
/** Whether to enable disaggregated prefill and decode execution */
bool enable_disagg_pd;
/** Whether to enable online-offline co-location in disaggregated PD mode */
bool enable_pd_ooc;
/** Whether to enable schedule overlap for parallel execution */
bool enable_schedule_overlap;
/** Whether to enable shared memory for model execution */
bool enable_shm;
/** Whether to enable graph execution for REC */
bool enable_graph;
/** Whether to enable REC fast sampler */
bool enable_rec_fast_sampler;
/** Whether to enable prefill piecewise graph for REC */
bool enable_prefill_piecewise_graph;
/** Whether to enable xattention one-stage execution for REC */
bool enable_xattention_one_stage;
/** Whether to enable graph-mode decode without padding for REC */
bool enable_graph_mode_decode_no_padding;
/** Whether to enable block copy kernel */
bool enable_block_copy_kernel;
/** Whether to keep REC top-k outputs sorted */
bool enable_topk_sorted;
/** Whether to enable rec prefill only */
bool enable_rec_prefill_only;
/** KVCache transfer listen port */
uint32_t transfer_listen_port;
/** Number of multi-nodes in distributed deployment */
uint32_t nnodes;
/** Node rank in distributed deployment */
uint32_t node_rank;
/** Data parallel size for MLA attention */
uint32_t dp_size;
/** Expert parallel size for MoE model */
uint32_t ep_size;
/** Number of slots per kv cache block */
uint32_t block_size;
/** Max GPU memory size for kv cache (0 = auto-calculate available memory) */
uint32_t max_cache_size;
/** Max number of tokens per batch */
uint32_t max_tokens_per_batch;
/** Max number of sequences per batch */
uint32_t max_seqs_per_batch;
/** Max number of token per chunk in prefill stage */
uint32_t max_tokens_per_chunk_for_prefill;
/** Number of speculative tokens for speculative decoding */
uint32_t num_speculative_tokens;
/** Number of threads for handling input requests */
uint32_t num_request_handling_threads;
/** Expert parallel degree for MoE model */
uint32_t expert_parallel_degree;
/** Index ID for internal server ID (unique for multiple models/versions) */
uint32_t server_idx;
/** Beam width for beam search decoding (1 for greedy search) */
uint32_t beam_width;
/** Maximum number of decode rounds for each inference request */
uint32_t max_decode_rounds;
/** Maximum number of tokens allowed per inference request */
uint32_t max_token_per_req;
/** Maximum GPU memory utilization ratio for model inference */
float max_memory_utilization;
/** Maximum REC worker pipeline concurrency */
uint32_t rec_worker_max_concurrency;
/** Model task type (generate/embed) */
char task[XLLM_META_STRING_FIELD_MAX_LEN];
/** NPU communication backend (lccl/hccl). Use hccl when dp is enabled */
char communication_backend[XLLM_META_STRING_FIELD_MAX_LEN];
/** Instance role (DEFAULT/PREFILL/DECODE/MIX) */
char instance_role[XLLM_META_STRING_FIELD_MAX_LEN];
/** Device IP address for NPU communication */
char device_ip[XLLM_META_STRING_FIELD_MAX_LEN];
/** Master address for multi-node distributed serving (e.g. 10.18.1.1:9999) */
char master_node_addr[XLLM_META_STRING_FIELD_MAX_LEN];
/** XService server address (empty string = disable XService) */
char xservice_addr[XLLM_META_STRING_FIELD_MAX_LEN];
/** Unique instance name for identification */
char instance_name[XLLM_META_STRING_FIELD_MAX_LEN];
/** KV cache transfer mode (PUSH/PULL) */
char kv_cache_transfer_mode[XLLM_META_STRING_FIELD_MAX_LEN];
/** Log directory path (empty string = disable logging) */
char log_dir[XLLM_META_STRING_FIELD_MAX_LEN];
/** Draft hf model path (empty string = no draft model) */
char draft_model[XLLM_META_STRING_FIELD_MAX_LEN];
/**
* Devices to run the draft model on (e.g. npu:0, npu:0,npu:1).
* Empty string = use the same devices as main model
*/
char draft_devices[XLLM_META_STRING_FIELD_MAX_LEN];
} XLLM_InitLLMOptions;
/**
* @brief Chat message structure (for ChatCompletions)
*/
typedef struct XLLM_CAPI_EXPORT XLLM_ChatMessage {
/** Message role (system/user/assistant) */
char role[XLLM_META_STRING_FIELD_MAX_LEN];
/** Message content (NULL for function call messages) */
char* content;
} XLLM_ChatMessage;
/**
* @brief Inference request parameters
* @note All numeric fields are fixed-width integers with value ranges defined
* in macros;
*/
typedef struct XLLM_CAPI_EXPORT XLLM_RequestParams {
/** Whether to include original prompt in response */
bool echo;
/** Whether it's an offline request */
bool offline;
/** Whether to return token log probabilities */
bool logprobs;
/** Whether to ignore EOS token */
bool ignore_eos;
/** Number of completions to return per prompt */
uint32_t n;
/** Maximum number of tokens to generate. Must be <= model context length */
uint32_t max_tokens;
/** Number of sequences to generate per prompt for top-n selection */
uint32_t best_of;
/** SLO timeout in milliseconds (0 = unlimited) */
int32_t ttlt_slo_ms;
int32_t ttft_slo_ms;
int32_t tpot_slo_ms;
/** Beam search width (0 = disable beam search) */
uint32_t beam_width;
/** Final number of beam search results to return (0 = use beam_width) */
uint32_t num_return_sequences;
/** Number of top log probabilities to return */
int64_t top_logprobs;
/** Top-K sampling cutoff (-1 = 0xFFFFFFFF means disabled) */
int64_t top_k;
/** Top-P sampling cutoff (range: [0.0, 1.0]) */
float top_p;
/** Frequency penalty (range: [0.0, 2.0]) */
float frequency_penalty;
/** Presence penalty (range: [-2.0, 2.0]) */
float presence_penalty;
/** Repetition penalty. >1.0 encourages new tokens, <1.0 encourages repetition
*/
float repetition_penalty;
/** Sampling temperature (range: [0.0, 2.0]) */
float temperature;
/** Request id */
char request_id[XLLM_META_STRING_FIELD_MAX_LEN];
} XLLM_RequestParams;
/**
* @brief API response status codes
*/
typedef enum XLLM_CAPI_EXPORT XLLM_StatusCode {
/** Request succeeded */
kSuccess = 0,
/** LLM instance not initialized */
kNotInitialized = 1,
/** Specified model ID not loaded */
kModelNotFound = 2,
/** Request timed out */
kTimeout = 3,
/** Invalid input parameters */
kInvalidRequest = 4,
/** Internal system error */
kInternalError = 5
} XLLM_StatusCode;
/**
* @brief Token usage statistics for inference request
*/
typedef struct XLLM_CAPI_EXPORT XLLM_Usage {
/** Number of tokens in the prompt */
int32_t prompt_tokens;
/** Number of tokens in the generated completion */
int32_t completion_tokens;
/** Total tokens used (prompt + completion) */
int32_t total_tokens;
} XLLM_Usage;
/**
* @brief Token log probability structure
*/
typedef struct XLLM_CAPI_EXPORT XLLM_LogProb {
/** Token ID */
uint32_t token_id;
/** Log probability of the token */
float logprob;
} XLLM_LogProb;
/**
* @brief List of token log probabilities
*/
typedef struct XLLM_CAPI_EXPORT XLLM_LogProbs {
/** Pointer to array of log probability entries */
XLLM_LogProb* entries;
/** Number of entries in the logprobs array */
size_t entries_size;
} XLLM_LogProbs;
/**
* @brief Inference result candidate
*/
typedef struct XLLM_CAPI_EXPORT XLLM_Choice {
/** Index of the generated completion candidate */
uint32_t index;
/** Generated text for completions inference (NULL for Chat mode) */
char* text;
/** Generated message for chatcompletions inference (NULL for Completion mode)
*/
XLLM_ChatMessage* message;
/** Generated token ids */
int32_t* token_ids;
/** Generated token ids size */
size_t token_size;
/** Token log probabilities */
XLLM_LogProbs logprobs;
/** Reason generation stopped (stop/length/function_call) */
char finish_reason[XLLM_META_STRING_FIELD_MAX_LEN];
} XLLM_Choice;
/**
* @brief List of inference result candidates
*/
typedef struct XLLM_CAPI_EXPORT XLLM_Choices {
/** Pointer to array of completion choice entries */
XLLM_Choice* entries;
/** Number of entries in the choices array */
size_t entries_size;
} XLLM_Choices;
/**
* @brief REC/OneRec specific output extension aligned by choice index
*/
typedef struct XLLM_CAPI_EXPORT XLLM_RecOutput {
/** Choice index this REC extension belongs to */
uint32_t index;
/** Selected REC item ids for this choice */
int64_t* item_ids;
/** Number of item ids in the item_ids array */
size_t item_ids_size;
/** Token-aligned REC/OneRec logprobs for this choice */
float* rec_token_logprobs;
/** Number of entries in rec_token_logprobs */
size_t rec_token_logprobs_size;
} XLLM_RecOutput;
/**
* @brief List of REC/OneRec specific output extensions
*/
typedef struct XLLM_CAPI_EXPORT XLLM_RecOutputs {
/** Pointer to array of REC output entries */
XLLM_RecOutput* entries;
/** Number of entries in the REC output array */
size_t entries_size;
} XLLM_RecOutputs;
#define XLLM_ERROR_INFO_MAX_LEN 512
/**
* @brief Inference response structure
*/
typedef struct XLLM_CAPI_EXPORT XLLM_Response {
/** Response status code (0 = success, non-zero = error) */
XLLM_StatusCode status_code;
/** Error details (NULL = no error) */
char error_info[XLLM_ERROR_INFO_MAX_LEN];
/** Unique ID for the completion request (fixed-length string) */
char id[XLLM_META_STRING_FIELD_MAX_LEN];
/** Object type (fixed to "text_completion") */
char object[XLLM_META_STRING_FIELD_MAX_LEN];
/** Unix timestamp (seconds) of when the completion was created */
int64_t created;
/** Model name used for the completion */
char model[XLLM_META_STRING_FIELD_MAX_LEN];
/** List of generated completion candidates */
XLLM_Choices choices;
/** Token usage statistics for the request */
XLLM_Usage usage;
/** REC/OneRec specific response extensions */
XLLM_RecOutputs rec_outputs;
} XLLM_Response;
/**
* @brief Enumeration of tensor data types
*/
typedef enum XLLM_CAPI_EXPORT XLLM_DataType {
XLLM_DTYPE_UNDEFINED = 0,
XLLM_DTYPE_FLOAT16 = 1,
XLLM_DTYPE_FLOAT32 = 2,
XLLM_DTYPE_FLOAT64 = 3,
XLLM_DTYPE_BFLOAT16 = 4,
XLLM_DTYPE_INT8 = 5,
XLLM_DTYPE_INT16 = 6,
XLLM_DTYPE_INT32 = 7,
XLLM_DTYPE_INT64 = 8,
XLLM_DTYPE_UINT8 = 9,
XLLM_DTYPE_UINT16 = 10,
XLLM_DTYPE_UINT32 = 11,
XLLM_DTYPE_UINT64 = 12,
XLLM_DTYPE_BOOL = 13,
XLLM_DTYPE_STRING = 14
} XLLM_DataType;
/**
* @brief Structure representing tensor dimensions (shape)
* @note Max supported rank is 8 (matches dim array length)
*/
typedef struct XLLM_CAPI_EXPORT XLLM_Dims {
/** Number of dimensions (0=scalar, 1=vector, ..., 8) */
int rank;
/** Size of each dimension (unused dims must be 0) */
int dim[8];
} XLLM_Dims;
/**
* @brief Core tensor structure for numerical computation
* @warning 1. data pointer is read-only, managed by external caller
* 2. dtype must match the actual type of data buffer
* 3. dims.rank must not exceed 8
*/
typedef struct XLLM_CAPI_EXPORT XLLM_Tensor {
/** Data type of tensor elements */
XLLM_DataType dtype;
/** Dimension information (shape) of the tensor */
XLLM_Dims dims;
/** Read-only pointer to tensor data buffer */
const void* data;
} XLLM_Tensor;
/**
* @brief Dynamic list of tensors (replaces C++ std::vector<XLLM_Tensor>)
*/
typedef struct XLLM_CAPI_EXPORT XLLM_Tensors {
XLLM_Tensor* entries;
size_t entries_size;
} XLLM_Tensors;
/**
* @brief Enumeration of multimodal data types (bitmask compatible)
* @note Each type is a bit flag (supports multiple types via type_mask)
*/
typedef enum XLLM_CAPI_EXPORT XLLM_MM_Type {
/** No multimodal type (invalid state) */
XLLM_MM_TYPE_NONE = 0,
/** Image modality (JPG/PNG/BMP) */
XLLM_MM_TYPE_IMAGE = 1 << 0,
/** Audio modality (WAV/MP3) */
XLLM_MM_TYPE_AUDIO = 1 << 1,
/** Video modality (H264/H265) */
XLLM_MM_TYPE_VIDEO = 1 << 2,
/** Text modality (tokenized text) */
XLLM_MM_TYPE_TEXT = 1 << 3,
/** Embedding modality (token embeddings) */
XLLM_MM_TYPE_EMBEDDING = 1 << 4
} XLLM_MM_Type;
/**
* @brief Multimodal value (variant type: single tensor or tensor list)
*/
typedef struct XLLM_CAPI_EXPORT XLLM_MM_Value {
/** Type flag: true=single tensor, false=tensor list */
bool is_single_tensor;
union {
/** Single tensor (valid if is_single_tensor=true) */
XLLM_Tensor tensor;
/** Tensor list (valid if is_single_tensor=false) */
XLLM_Tensors tensors;
} data;
} XLLM_MM_Value;
/**
* @brief Single entry in multimodal dictionary (key-value pair)
* @note 1. Key is fixed-length string (null-terminated if shorter than max len)
* 2. Key must be unique within a dictionary
*/
typedef struct XLLM_CAPI_EXPORT XLLM_MM_DictEntry {
/** Fixed-length key */
char key[XLLM_META_STRING_FIELD_MAX_LEN];
/** Value associated with the key */
XLLM_MM_Value value;
} XLLM_MM_DictEntry;
/**
* @brief Multimodal dictionary (array of key-value entries)
* @note 1. entries is a heap-allocated array (must be freed by caller)
* 2. entries_size = number of valid entries (no empty slots)
*/
typedef struct XLLM_CAPI_EXPORT XLLM_MM_Dict {
XLLM_MM_DictEntry* entries;
size_t entries_size;
} XLLM_MM_Dict;
/**
* @brief Token position information (offset + length) for multimodal data
* @note Used to map multimodal data to token positions in sequence
*/
typedef struct XLLM_CAPI_EXPORT XLLM_MM_TokenPos {
/** Start offset of tokens (0-based) */
uint32_t offset;
/** Number of tokens (must be >0 for valid position) */
uint32_t length;
} XLLM_MM_TokenPos;
/**
* @brief Base struct for multimodal metadata (to be extended by specific
* modalities)
* @note This is a placeholder for modality-specific metadata (e.g., image size,
* audio sample rate) Extend with union for image/audio/video metadata in
* production use
*/
typedef struct XLLM_CAPI_EXPORT XLLM_MM_Meta {
// Placeholder for future extension,
// e.g., XLLM_ImageMeta|XLLM_AudioMeta|XLLM_VideoMeta
} XLLM_MM_Meta;
/**
* @brief State information for a single multimodal item
*/
typedef struct XLLM_CAPI_EXPORT XLLM_MM_State {
/** Token position for multimodal data alignment */
XLLM_MM_TokenPos token_pos;
} XLLM_MM_State;
/**
* @brief Single multimodal data item (core unit of multimodal data)
* @note 1. type = modality type (image/audio/video/text/embedding)
* 2. data = numerical content (tensor/tensor list)
* 3. meta = modality-specific metadata (empty in base version)
* 4. state = token position and processing state
*/
typedef struct XLLM_CAPI_EXPORT XLLM_MM_Item {
/** Modality type (e.g., XLLM_MM_TYPE_EMBEDDING) */
XLLM_MM_Type type;
/** Core data (tensor/tensor list) */
XLLM_MM_Value data;
/** Modality-specific metadata (extendable) */
XLLM_MM_Meta meta;
/** Processing state and token position */
XLLM_MM_State state;
} XLLM_MM_Item;
/**
* @brief List of multimodal items (replaces C++ std::vector<XLLM_MM_Item>)
* @note 1. entries is a heap-allocated array (must be freed by caller)
* 2. entries_size = number of valid items (no empty slots)
*/
typedef struct XLLM_CAPI_EXPORT XLLM_MM_Items {
XLLM_MM_Item* entries;
size_t entries_size;
} XLLM_MM_Items;
/**
* @brief Core multimodal data container (supports list/dict storage)
*/
typedef struct XLLM_CAPI_EXPORT XLLM_MM_Data {
/** Bitmask of multimodal types (e.g., IMAGE | EMBEDDING) */
uint32_t type_mask;
/** Storage type: true=XLLM_MM_Dict, false=XLLM_MM_Items */
bool is_dict;
union {
/** Dict storage (valid if is_dict=true) */
XLLM_MM_Dict dict;
/** List storage (valid if is_dict=false) */
XLLM_MM_Items items;
} data;
} XLLM_MM_Data;
#ifdef __cplusplus
}
#endif
#endif // XLLM_C_TYPES_H