router: Support parallel sampling num > 1 in grpc_server and non-stream handling (#10929)
This commit is contained in:
@@ -103,6 +103,7 @@ impl SglangSchedulerClient {
|
||||
logprob_start_len: -1,
|
||||
top_logprobs_num: body.top_logprobs.unwrap_or(0) as i32,
|
||||
return_hidden_states: body.return_hidden_states,
|
||||
stream: body.stream,
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
@@ -367,14 +368,14 @@ mod tests {
|
||||
#[test]
|
||||
fn test_generate_stream_chunk() {
|
||||
let chunk = proto::GenerateStreamChunk {
|
||||
token_id: 1234,
|
||||
token_ids: vec![1234, 5678],
|
||||
prompt_tokens: 5,
|
||||
completion_tokens: 2,
|
||||
cached_tokens: 3,
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
assert_eq!(chunk.token_id, 1234);
|
||||
assert_eq!(chunk.token_ids, vec![1234, 5678]);
|
||||
assert_eq!(chunk.prompt_tokens, 5);
|
||||
assert_eq!(chunk.completion_tokens, 2);
|
||||
assert_eq!(chunk.cached_tokens, 3);
|
||||
|
||||
Reference in New Issue
Block a user