[router] remove all tokenizer metrics for performance (#9474)

2025-08-21 18:35:24 -07:00
parent 9708d353b7
commit 53e2cd464f
8 changed files with 28 additions and 117 deletions
--- a/sgl-router/src/tokenizer/tests.rs
+++ b/sgl-router/src/tokenizer/tests.rs
@@ -129,9 +129,7 @@ fn test_thread_safety() {
            thread::spawn(move || {
                let text = "Hello test".to_string();
                let encoding = tokenizer_clone.encode(&text).unwrap();
-                let decoded = tokenizer_clone
-                    .decode(&encoding.token_ids(), false)
-                    .unwrap();
+                let decoded = tokenizer_clone.decode(encoding.token_ids(), false).unwrap();
                assert!(decoded.contains("Hello") || decoded.contains("test"));
                i
            })