[router] add grpc router pd mode for chat and generate (#11140)

This commit is contained in:
Simo Lin
2025-10-04 09:58:28 -04:00
committed by GitHub
parent ffd03a9bd3
commit d736e0b65e
11 changed files with 3169 additions and 1080 deletions

View File

@@ -342,6 +342,12 @@ impl SglangSchedulerClient {
.map_err(|_| "min_tokens must fit into a 32-bit signed integer".to_string())?;
}
// Handle n with conversion
if let Some(n) = p.n {
sampling.n = i32::try_from(n)
.map_err(|_| "n must fit into a 32-bit signed integer".to_string())?;
}
// Handle constraints (exactly one allowed)
sampling.constraint = Self::build_single_constraint_from_plain(p)?;