[router] add grpc router pd mode for chat and generate (#11140)

This commit is contained in:
Simo Lin
2025-10-04 09:58:28 -04:00
committed by GitHub
parent ffd03a9bd3
commit d736e0b65e
11 changed files with 3169 additions and 1080 deletions

View File

@@ -2,6 +2,11 @@ use serde::{Deserialize, Serialize};
use serde_json::{to_value, Map, Number, Value};
use std::collections::HashMap;
// Default model value when not specified
fn default_model() -> String {
"unknown".to_string()
}
// # Protocol Specifications
//
// This module contains all protocol definitions for OpenAI and SGLang APIs.
@@ -169,6 +174,7 @@ pub struct ChatCompletionRequest {
pub messages: Vec<ChatMessage>,
/// ID of the model to use
#[serde(default = "default_model")]
pub model: String,
/// Number between -2.0 and 2.0. Positive values penalize new tokens based on their existing frequency in the text so far