EngineX-Hygon/sglang
5
0
Fork 0
Code Issues Pull Requests Actions 7 Projects Releases Wiki Activity

[router] remove all tokenizer metrics for performance (#9474)

Browse Source
This commit is contained in:
Chang Su
2025-08-21 18:35:24 -07:00
committed by GitHub
parent 9708d353b7
commit 53e2cd464f
8 changed files with 28 additions and 117 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
sgl-router/src/tokenizer/tiktoken.rs
View File

@@ -213,7 +213,7 @@ mod tests {
let text = "Hello, world!";
let encoding = tokenizer.encode(text).unwrap();
let decoded = tokenizer.decode(&encoding.token_ids(), false).unwrap();
let decoded = tokenizer.decode(encoding.token_ids(), false).unwrap();
assert_eq!(decoded, text);
}
@@ -226,7 +226,7 @@ mod tests {
assert_eq!(encodings.len(), 3);
for (i, encoding) in encodings.iter().enumerate() {
let decoded = tokenizer.decode(&encoding.token_ids(), false).unwrap();
let decoded = tokenizer.decode(encoding.token_ids(), false).unwrap();
assert_eq!(decoded, texts[i]);
}
}