[router] add tiktokenizer and sequence in router (#9354)

Co-authored-by: Chang Su <chang.s.su@oracle.com>
This commit is contained in:
Simo Lin
2025-08-19 10:46:28 -07:00
committed by GitHub
parent a3b810ebdb
commit 0b95a01a8f
7 changed files with 578 additions and 4 deletions

View File

@@ -129,7 +129,9 @@ fn test_thread_safety() {
thread::spawn(move || {
let text = "Hello test".to_string();
let encoding = tokenizer_clone.encode(&text).unwrap();
let decoded = tokenizer_clone.decode(encoding.token_ids(), false).unwrap();
let decoded = tokenizer_clone
.decode(&encoding.token_ids(), false)
.unwrap();
assert!(decoded.contains("Hello") || decoded.contains("test"));
i
})