002f9879b2
Replaces cherry-picked upstream_ref with complete source trees. xllm/ — Iluvatar official C++ inference engine (15MB, 1470 files) Complete: kernels → layers → models → runtime → scheduler → api Excluded: .git, binary images, third_party submodule checkouts ds_vllm/ — Iluvatar official vllm fork (8MB, 703 files) Included: csrc/ (ALL CUDA kernels), fused_moe/, qwen3_5 model, _custom_ops Excluded: tests, benchmarks, docs, examples (not needed for reference) Critical call chains now fully traceable: MoE: moe_topk_softmax_kernels.cuh → ixformer.h → fused_moe.cpp → layer GDN: qwen3_gated_delta_net_base.cpp → qwen3_5_gated_delta_net.cpp Attention: ixformer.h → xllm_paged_attention → attention.cpp
104 lines
2.5 KiB
Protocol Buffer
104 lines
2.5 KiB
Protocol Buffer
syntax = "proto3";
|
|
|
|
package xllm_service.proto;
|
|
option cc_generic_services = true;
|
|
|
|
import "disagg_pd.proto";
|
|
|
|
message Empty {}
|
|
|
|
message Status {
|
|
bool ok = 1;
|
|
}
|
|
|
|
message StatusCode {
|
|
int32 status_code = 1;
|
|
}
|
|
|
|
enum InstanceType {
|
|
DEFAULT = 0;
|
|
PREFILL = 1;
|
|
DECODE = 2;
|
|
MIX = 3;
|
|
}
|
|
|
|
message InstanceMetaInfo {
|
|
// http server address currently
|
|
string name = 1;
|
|
// rpc server address
|
|
string rpc_address = 2;
|
|
optional InstanceType type = 3;
|
|
repeated uint64 cluster_ids = 4;
|
|
repeated int64 k_cache_ids = 5;
|
|
repeated int64 v_cache_ids = 6;
|
|
int32 dp_size = 7;
|
|
repeated string addrs = 8;
|
|
repeated string device_ips = 9;
|
|
repeated uint32 ports = 10;
|
|
string incarnation_id = 11;
|
|
uint64 register_ts_ms = 12;
|
|
}
|
|
|
|
message KvCacheEvent {
|
|
repeated bytes stored_cache = 1;
|
|
repeated bytes removed_cache = 2;
|
|
}
|
|
|
|
message LoadMetrics {
|
|
uint64 waiting_requests_num = 1;
|
|
float gpu_cache_usage_perc = 2;
|
|
}
|
|
|
|
message LatencyMetrics {
|
|
int64 recent_max_ttft = 1;
|
|
int64 recent_max_tbt = 2;
|
|
}
|
|
|
|
// XTensor mode info for heartbeat updates
|
|
message XTensorHeartbeatInfo {
|
|
// Per-worker free physical pages (index = worker rank)
|
|
repeated uint64 worker_free_phy_pages = 1;
|
|
// Model weight segments in GlobalXTensor (key: model_id)
|
|
// Ordered segments that concatenate to form the full model weights
|
|
map<string, WeightSegmentList> model_weight_segments = 3;
|
|
}
|
|
|
|
// A single weight segment in GlobalXTensor
|
|
message WeightSegmentInfo {
|
|
uint64 offset = 1; // Byte offset from GlobalXTensor base
|
|
uint64 size = 2; // Segment size in bytes
|
|
}
|
|
|
|
// List of weight segments for a model
|
|
message WeightSegmentList {
|
|
repeated WeightSegmentInfo segments = 1;
|
|
}
|
|
|
|
message HeartbeatRequest {
|
|
string name = 1;
|
|
KvCacheEvent cache_event = 2;
|
|
LoadMetrics load_metrics = 3;
|
|
LatencyMetrics latency_metrics = 4;
|
|
XTensorHeartbeatInfo xtensor_info = 5;
|
|
string incarnation_id = 6;
|
|
}
|
|
|
|
message InstanceID {
|
|
string name = 1;
|
|
}
|
|
|
|
message InstanceIDs {
|
|
repeated string names = 1;
|
|
}
|
|
|
|
service XllmRpcService {
|
|
rpc RegisterInstance(InstanceMetaInfo) returns (StatusCode) {}
|
|
rpc GetInstanceInfo(InstanceID) returns (InstanceMetaInfo) {}
|
|
rpc Heartbeat(HeartbeatRequest) returns (Status) {}
|
|
rpc GetStaticDecodeList(InstanceID) returns (InstanceIDs) {}
|
|
rpc GetStaticPrefillList(InstanceID) returns (InstanceIDs) {}
|
|
// xllm service receive response from decode instance directly in disagg pd mode.
|
|
// This can eliminate the cost brought by forwarding through prefill.
|
|
rpc Generations(xllm.proto.DisaggStreamGenerations) returns (xllm.proto.StatusSet) {}
|
|
}
|