[router] update generate spec to align with sgl io struct (#11591)
This commit is contained in:
@@ -877,7 +877,7 @@ impl ResponseProcessingStage {
|
||||
}
|
||||
|
||||
// Non-streaming: Delegate to ResponseProcessor
|
||||
let request_logprobs = ctx.generate_request().return_logprob;
|
||||
let request_logprobs = ctx.generate_request().return_logprob.unwrap_or(false);
|
||||
let generate_request = ctx.generate_request_arc();
|
||||
|
||||
let stop_decoder = ctx
|
||||
|
||||
@@ -616,7 +616,7 @@ impl StreamingProcessor {
|
||||
generate_request: Arc<GenerateRequest>,
|
||||
dispatch: context::DispatchMetadata,
|
||||
) -> Response {
|
||||
let return_logprob = generate_request.return_logprob;
|
||||
let return_logprob = generate_request.return_logprob.unwrap_or(false);
|
||||
|
||||
// Create SSE channel
|
||||
let (tx, rx) = mpsc::unbounded_channel::<Result<Bytes, io::Error>>();
|
||||
|
||||
@@ -150,11 +150,6 @@ impl PDRouter {
|
||||
}
|
||||
|
||||
fn get_generate_batch_size(req: &GenerateRequest) -> Option<usize> {
|
||||
if let Some(StringOrArray::Array(arr)) = &req.prompt {
|
||||
if !arr.is_empty() {
|
||||
return Some(arr.len());
|
||||
}
|
||||
}
|
||||
if let Some(text) = &req.text {
|
||||
if text.contains("[") && text.contains("]") {
|
||||
return None;
|
||||
@@ -1061,18 +1056,10 @@ impl RouterTrait for PDRouter {
|
||||
model_id: Option<&str>,
|
||||
) -> Response {
|
||||
let is_stream = body.stream;
|
||||
let return_logprob = body.return_logprob;
|
||||
let return_logprob = body.return_logprob.unwrap_or(false);
|
||||
|
||||
let request_text = if self.policies_need_request_text() {
|
||||
body.text
|
||||
.as_deref()
|
||||
.or_else(|| {
|
||||
body.prompt.as_ref().and_then(|p| match p {
|
||||
StringOrArray::String(s) => Some(s.as_str()),
|
||||
StringOrArray::Array(v) => v.first().map(|s| s.as_str()),
|
||||
})
|
||||
})
|
||||
.map(|s| s.to_string())
|
||||
body.text.as_deref().map(|s| s.to_string())
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
Reference in New Issue
Block a user