[router] update generate spec to align with sgl io struct (#11591)

This commit is contained in:
Simo Lin
2025-10-14 02:51:33 -04:00
committed by GitHub
parent 27ef1459e6
commit 0b9915c132
7 changed files with 196 additions and 59 deletions

View File

@@ -877,7 +877,7 @@ impl ResponseProcessingStage {
}
// Non-streaming: Delegate to ResponseProcessor
let request_logprobs = ctx.generate_request().return_logprob;
let request_logprobs = ctx.generate_request().return_logprob.unwrap_or(false);
let generate_request = ctx.generate_request_arc();
let stop_decoder = ctx

View File

@@ -616,7 +616,7 @@ impl StreamingProcessor {
generate_request: Arc<GenerateRequest>,
dispatch: context::DispatchMetadata,
) -> Response {
let return_logprob = generate_request.return_logprob;
let return_logprob = generate_request.return_logprob.unwrap_or(false);
// Create SSE channel
let (tx, rx) = mpsc::unbounded_channel::<Result<Bytes, io::Error>>();

View File

@@ -150,11 +150,6 @@ impl PDRouter {
}
fn get_generate_batch_size(req: &GenerateRequest) -> Option<usize> {
if let Some(StringOrArray::Array(arr)) = &req.prompt {
if !arr.is_empty() {
return Some(arr.len());
}
}
if let Some(text) = &req.text {
if text.contains("[") && text.contains("]") {
return None;
@@ -1061,18 +1056,10 @@ impl RouterTrait for PDRouter {
model_id: Option<&str>,
) -> Response {
let is_stream = body.stream;
let return_logprob = body.return_logprob;
let return_logprob = body.return_logprob.unwrap_or(false);
let request_text = if self.policies_need_request_text() {
body.text
.as_deref()
.or_else(|| {
body.prompt.as_ref().and_then(|p| match p {
StringOrArray::String(s) => Some(s.as_str()),
StringOrArray::Array(v) => v.first().map(|s| s.as_str()),
})
})
.map(|s| s.to_string())
body.text.as_deref().map(|s| s.to_string())
} else {
None
};