[router] introducing tokenizer trait (#9287)

2025-08-17 16:30:01 -07:00
parent a1c7f742f9
commit ff0cf51c8e
7 changed files with 502 additions and 0 deletions
--- a/sgl-router/Cargo.toml
+++ b/sgl-router/Cargo.toml
@@ -43,6 +43,8 @@ uuid = { version = "1.10", features = ["v4", "serde"] }
 thiserror = "2.0.12"
 url = "2.5.4"
 tokio-stream = { version = "0.1", features = ["sync"] }
+anyhow = "1.0"
+tokenizers = "0.21.4"

 [dev-dependencies]
 criterion = { version = "0.5", features = ["html_reports"] }