[router] complete router oai spec (#8828)
This commit is contained in:
Simo Lin
@@ -8,14 +8,118 @@ use sglang_router_rs::openai_api_types::{
|
||||
};
|
||||
use sglang_router_rs::routers::request_adapter::{RouteableRequest, ToPdRequest};
|
||||
|
||||
/// Create a default GenerateRequest for benchmarks with minimal fields set
|
||||
fn default_generate_request() -> GenerateRequest {
|
||||
GenerateRequest {
|
||||
text: None,
|
||||
prompt: None,
|
||||
input_ids: None,
|
||||
stream: false,
|
||||
parameters: None,
|
||||
sampling_params: None,
|
||||
return_logprob: false,
|
||||
// SGLang Extensions
|
||||
lora_path: None,
|
||||
session_params: None,
|
||||
return_hidden_states: false,
|
||||
rid: None,
|
||||
}
|
||||
}
|
||||
|
||||
/// Create a default ChatCompletionRequest for benchmarks with minimal fields set
|
||||
fn default_chat_completion_request() -> ChatCompletionRequest {
|
||||
ChatCompletionRequest {
|
||||
model: String::new(),
|
||||
messages: vec![],
|
||||
max_tokens: None,
|
||||
max_completion_tokens: None,
|
||||
temperature: None,
|
||||
top_p: None,
|
||||
n: None,
|
||||
stream: false,
|
||||
stream_options: None,
|
||||
stop: None,
|
||||
presence_penalty: None,
|
||||
frequency_penalty: None,
|
||||
logit_bias: None,
|
||||
logprobs: false,
|
||||
top_logprobs: None,
|
||||
user: None,
|
||||
response_format: None,
|
||||
seed: None,
|
||||
tools: None,
|
||||
tool_choice: None,
|
||||
parallel_tool_calls: None,
|
||||
function_call: None,
|
||||
functions: None,
|
||||
// SGLang Extensions
|
||||
top_k: None,
|
||||
min_p: None,
|
||||
min_tokens: None,
|
||||
repetition_penalty: None,
|
||||
regex: None,
|
||||
ebnf: None,
|
||||
stop_token_ids: None,
|
||||
no_stop_trim: false,
|
||||
ignore_eos: false,
|
||||
continue_final_message: false,
|
||||
skip_special_tokens: true,
|
||||
// SGLang Extensions
|
||||
lora_path: None,
|
||||
session_params: None,
|
||||
separate_reasoning: true,
|
||||
stream_reasoning: true,
|
||||
return_hidden_states: false,
|
||||
}
|
||||
}
|
||||
|
||||
/// Create a default CompletionRequest for benchmarks with minimal fields set
|
||||
fn default_completion_request() -> CompletionRequest {
|
||||
CompletionRequest {
|
||||
model: String::new(),
|
||||
prompt: StringOrArray::String(String::new()),
|
||||
suffix: None,
|
||||
max_tokens: None,
|
||||
temperature: None,
|
||||
top_p: None,
|
||||
n: None,
|
||||
stream: false,
|
||||
stream_options: None,
|
||||
logprobs: None,
|
||||
echo: false,
|
||||
stop: None,
|
||||
presence_penalty: None,
|
||||
frequency_penalty: None,
|
||||
best_of: None,
|
||||
logit_bias: None,
|
||||
user: None,
|
||||
seed: None,
|
||||
// SGLang Extensions
|
||||
top_k: None,
|
||||
min_p: None,
|
||||
min_tokens: None,
|
||||
repetition_penalty: None,
|
||||
regex: None,
|
||||
ebnf: None,
|
||||
json_schema: None,
|
||||
stop_token_ids: None,
|
||||
no_stop_trim: false,
|
||||
ignore_eos: false,
|
||||
skip_special_tokens: true,
|
||||
// SGLang Extensions
|
||||
lora_path: None,
|
||||
session_params: None,
|
||||
return_hidden_states: false,
|
||||
other: serde_json::Map::new(),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_benchmark_request_creation() {
|
||||
// Ensure all benchmark request types can be created without panicking
|
||||
|
||||
let generate_req = GenerateRequest {
|
||||
text: Some("Test prompt".to_string()),
|
||||
input_ids: None,
|
||||
prompt: None,
|
||||
parameters: Some(GenerateParameters {
|
||||
max_new_tokens: Some(100),
|
||||
temperature: Some(0.8),
|
||||
@@ -33,8 +137,7 @@ fn test_benchmark_request_creation() {
|
||||
repetition_penalty: Some(1.0),
|
||||
..Default::default()
|
||||
}),
|
||||
stream: false,
|
||||
return_logprob: false,
|
||||
..default_generate_request()
|
||||
};
|
||||
|
||||
let chat_req = ChatCompletionRequest {
|
||||
@@ -49,44 +152,23 @@ fn test_benchmark_request_creation() {
|
||||
temperature: Some(0.7),
|
||||
top_p: Some(1.0),
|
||||
n: Some(1),
|
||||
stream: false,
|
||||
stream_options: None,
|
||||
stop: None,
|
||||
presence_penalty: Some(0.0),
|
||||
frequency_penalty: Some(0.0),
|
||||
logit_bias: None,
|
||||
logprobs: false,
|
||||
top_logprobs: None,
|
||||
user: None,
|
||||
response_format: None,
|
||||
seed: None,
|
||||
tools: None,
|
||||
tool_choice: None,
|
||||
parallel_tool_calls: Some(true),
|
||||
function_call: None,
|
||||
functions: None,
|
||||
..default_chat_completion_request()
|
||||
};
|
||||
|
||||
let completion_req = CompletionRequest {
|
||||
model: "test-model".to_string(),
|
||||
prompt: StringOrArray::String("Test prompt".to_string()),
|
||||
suffix: None,
|
||||
max_tokens: Some(50),
|
||||
temperature: Some(0.8),
|
||||
top_p: Some(1.0),
|
||||
n: Some(1),
|
||||
stream: false,
|
||||
stream_options: None,
|
||||
logprobs: None,
|
||||
echo: false,
|
||||
stop: None,
|
||||
presence_penalty: Some(0.0),
|
||||
frequency_penalty: Some(0.0),
|
||||
best_of: Some(1),
|
||||
logit_bias: None,
|
||||
user: None,
|
||||
seed: None,
|
||||
other: serde_json::Map::new(),
|
||||
..default_completion_request()
|
||||
};
|
||||
|
||||
// Test serialization works
|
||||
@@ -101,12 +183,7 @@ fn test_benchmark_serialization_roundtrip() {
|
||||
|
||||
let generate_req = GenerateRequest {
|
||||
text: Some("Test prompt".to_string()),
|
||||
input_ids: None,
|
||||
prompt: None,
|
||||
parameters: None,
|
||||
sampling_params: None,
|
||||
stream: false,
|
||||
return_logprob: false,
|
||||
..default_generate_request()
|
||||
};
|
||||
|
||||
// Serialize and deserialize
|
||||
@@ -125,12 +202,7 @@ fn test_benchmark_request_adaptation() {
|
||||
|
||||
let generate_req = GenerateRequest {
|
||||
text: Some("Test prompt".to_string()),
|
||||
input_ids: None,
|
||||
prompt: None,
|
||||
parameters: None,
|
||||
sampling_params: None,
|
||||
stream: false,
|
||||
return_logprob: false,
|
||||
..default_generate_request()
|
||||
};
|
||||
|
||||
let chat_req = ChatCompletionRequest {
|
||||
@@ -145,44 +217,23 @@ fn test_benchmark_request_adaptation() {
|
||||
temperature: Some(0.7),
|
||||
top_p: Some(1.0),
|
||||
n: Some(1),
|
||||
stream: false,
|
||||
stream_options: None,
|
||||
stop: None,
|
||||
presence_penalty: Some(0.0),
|
||||
frequency_penalty: Some(0.0),
|
||||
logit_bias: None,
|
||||
logprobs: false,
|
||||
top_logprobs: None,
|
||||
user: None,
|
||||
response_format: None,
|
||||
seed: None,
|
||||
tools: None,
|
||||
tool_choice: None,
|
||||
parallel_tool_calls: Some(true),
|
||||
function_call: None,
|
||||
functions: None,
|
||||
..default_chat_completion_request()
|
||||
};
|
||||
|
||||
let completion_req = CompletionRequest {
|
||||
model: "test-model".to_string(),
|
||||
prompt: StringOrArray::String("Test prompt".to_string()),
|
||||
suffix: None,
|
||||
max_tokens: Some(50),
|
||||
temperature: Some(0.8),
|
||||
top_p: Some(1.0),
|
||||
n: Some(1),
|
||||
stream: false,
|
||||
stream_options: None,
|
||||
logprobs: None,
|
||||
echo: false,
|
||||
stop: None,
|
||||
presence_penalty: Some(0.0),
|
||||
frequency_penalty: Some(0.0),
|
||||
best_of: Some(1),
|
||||
logit_bias: None,
|
||||
user: None,
|
||||
seed: None,
|
||||
other: serde_json::Map::new(),
|
||||
..default_completion_request()
|
||||
};
|
||||
|
||||
// Test PD adaptation (should not panic)
|
||||
@@ -197,12 +248,7 @@ fn test_benchmark_regular_routing() {
|
||||
|
||||
let generate_req = GenerateRequest {
|
||||
text: Some("Test prompt".to_string()),
|
||||
input_ids: None,
|
||||
prompt: None,
|
||||
parameters: None,
|
||||
sampling_params: None,
|
||||
stream: false,
|
||||
return_logprob: false,
|
||||
..default_generate_request()
|
||||
};
|
||||
|
||||
// Test regular routing methods (should not panic)
|
||||
@@ -217,12 +263,7 @@ fn test_benchmark_performance_baseline() {
|
||||
|
||||
let generate_req = GenerateRequest {
|
||||
text: Some("Short test prompt".to_string()),
|
||||
input_ids: None,
|
||||
prompt: None,
|
||||
parameters: None,
|
||||
sampling_params: None,
|
||||
stream: false,
|
||||
return_logprob: false,
|
||||
..default_generate_request()
|
||||
};
|
||||
|
||||
// Serialization should be fast (< 1ms for simple requests)
|
||||
|
||||
Reference in New Issue
Block a user